// May only be called by the processing goroutine, in a transaction. func sendProposal(inst *coproto.Instruction) { // We must be leader. proposal, leader := store.Proposal() if leader != config.Id() || !amLeader { panic("tried to send accept messages while not leader") } // Send accept messages to all other core nodes. var accept coproto.Accept accept.Proposal = new(uint64) *accept.Proposal = proposal accept.Instruction = inst for _, node := range config.CoreNodes() { if node == config.Id() { continue } if len(connections[node]) != 0 { c := connections[node][0] c.SendProto(4, &accept) } } // Behave as if we received our own accept message. addAccept(config.Id(), &accept) }
// Must be called from the processing goroutine, inside a transaction. // Handle an accept message that we've decided to respond to. // Called directly when we send accepted messages to other nodes. func addAccept(node uint16, msg *coproto.Accept) { // Send an accepted message to all other core nodes reachable. var accepted coproto.Accepted accepted.Proposal = msg.Proposal accepted.Leader = new(uint32) *accepted.Leader = uint32(node) accepted.Instruction = msg.Instruction // We must do this before sending accepted messages. // We behave as if we received one from ourselves. if !addAccepted(config.Id(), &accepted) { // Required too large an instruction slots slice. // We have NOT accepted this accept message, // and must NOT send accepted messages. return } for _, node := range config.CoreNodes() { if node == config.Id() { continue } if len(connections[node]) > 0 { connections[node][0].SendProto(5, &accepted) } } }
// Must be called from inside a transaction. func makeRequest(changes []store.Change) *store.ChangeRequest { req := new(store.ChangeRequest) req.RequestEntity = uint64(config.Id()) req.RequestNode = config.Id() req.RequestId = store.AllocateRequestId() req.Changeset = changes return req }
// Create change request creating new session entity, // attached to user entity, this node ID attached to it. // And the session then set as transient. func makeNewSessionRequest(userId uint64) *store.ChangeRequest { chset := make([]store.Change, 5) // Create new entity. // Entity ID 1 now refers to this within the changeset. chset[0].TargetEntity = 1 chset[0].Key = "id" chset[0].Value = strconv.FormatUint(chset[0].TargetEntity, 10) // Make new entity a session entity. chset[1].TargetEntity = 1 chset[1].Key = "kind" chset[1].Value = "session" // Attach session entity to user. chset[2].TargetEntity = userId chset[2].Key = "attach 1" chset[2].Value = "true" // Attach node ID to session entity. idStr := strconv.FormatUint(uint64(config.Id()), 10) chset[3].TargetEntity = 1 chset[3].Key = "attach " + idStr chset[3].Value = "true" // Set session entity as transient. chset[4].TargetEntity = 1 chset[4].Key = "transient" chset[4].Value = "true" return makeRequest(chset) }
// Checks for "attach <id>" entries attaching this node ID to sessions we lack // a connection to. Starts a timer to delete them if present. // Must be called during a transaction, holding the session lock. func checkOrphanAttaches() { attachedTo := store.AllAttachedTo(uint64(config.Id())) for _, session := range attachedTo { if sessions[session] == nil { startOrphanTimeout(session) } } }
func SetProposal(newProposal uint64, newLeader uint16) { if intLeader == config.Id() && newLeader != intLeader { lastLeader = time.Now() } if newProposal != intProposal || newLeader != intLeader { intProposal = newProposal intLeader = newLeader log.Print("shared/store: new proposal and leader ", intProposal, " ", intLeader) for _, cb := range proposalCallbacks { cb() } } }
func handleMsg(node uint16, conn *connect.BaseConn, msg *baseproto.Message) { if *msg.MsgType == 2 { // Change Forward message. forward := new(chproto.ChangeForward) err := proto.Unmarshal(msg.Content, forward) if err != nil { conn.Close() } // If this is not a core node, // and the requesting node is not this node, // discard the message. if uint16(*forward.Request.RequestNode) != config.Id() && !config.IsCore() { return } // Send a Change Forward Ack message to the sender. ack := new(chproto.ChangeForwardAck) ack.RequestNode = forward.Request.RequestNode ack.RequestId = forward.Request.RequestId conn.SendProto(3, ack) // Send the forward message to our processing goroutine. receivedForwardCh <- receivedForward{forward: forward, node: node} } else if *msg.MsgType == 3 { // Change Forward Ack message. ack := new(chproto.ChangeForwardAck) err := proto.Unmarshal(msg.Content, ack) if err != nil { conn.Close() } // Send the ack message to our processing goroutine. receivedAckCh <- receivedAck{ack: ack, node: node} } else { // Unknown message. conn.Close() } }
func orphanTimeout(id uint64) { store.StartTransaction() defer store.EndTransaction() // If the timeout has been removed, do nothing. if namelessRemoveTimeouts[id] == nil { return } // Detach ourselves from the session. // TODO: Should make sure we only do this once. ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10) session := store.GetEntity(id) if session != nil { chset := make([]store.Change, 1) chset[0].TargetEntity = id chset[0].Key = ourAttachStr chset[0].Value = "" req := makeRequest(chset) go chrequest.Request(req) } }
// Create change request creating new session entity, // attached to user entity, this node ID attached to it. // And the session then set as transient. func makeNewUserRequest(username, pass, salt string, transient bool) *store.ChangeRequest { count := 10 if transient { // Add an extra change for setting the user as transient. count++ } chset := make([]store.Change, count) // Create new entity. // Entity ID 1 now refers to this within the changeset. chset[0].TargetEntity = 1 chset[0].Key = "id" chset[0].Value = strconv.FormatUint(chset[0].TargetEntity, 10) // Make new entity a user entity. chset[1].TargetEntity = 1 chset[1].Key = "kind" chset[1].Value = "user" // Set username. chset[2].TargetEntity = 1 chset[2].Key = "name username" chset[2].Value = username // Set password. chset[3].TargetEntity = 1 chset[3].Key = "auth password" chset[3].Value = pass // Set salt. chset[4].TargetEntity = 1 chset[4].Key = "auth salt" chset[4].Value = salt // Create new entity. // Entity ID 2 now refers to this within the changeset. chset[5].TargetEntity = 2 chset[5].Key = "id" chset[5].Value = strconv.FormatUint(chset[0].TargetEntity, 10) // Make new entity a session entity. chset[6].TargetEntity = 2 chset[6].Key = "kind" chset[6].Value = "session" // Attach session entity to user. chset[7].TargetEntity = 1 chset[7].Key = "attach 2" chset[7].Value = "true" // Attach node ID to session entity. idStr := strconv.FormatUint(uint64(config.Id()), 10) chset[8].TargetEntity = 2 chset[8].Key = "attach " + idStr chset[8].Value = "true" // Set session entity as transient. chset[9].TargetEntity = 2 chset[9].Key = "transient" chset[9].Value = "true" // Set user entity as transient, if the user is to be so. if transient { chset[10].TargetEntity = 1 chset[10].Key = "transient" chset[10].Value = "true" } return makeRequest(chset) }
// Main function for the handling goroutine for conn. func handleConn(conn *userConn) { // Do cleanup in a defer, so if they crash us we still tidy up here. // A small nod towards tolerating bad input, with much more needed. defer func() { // Remove the connection from our connection set if present. // It will not be present if we are degraded or similar. connectionsLock.Lock() if _, exists := connections[conn]; exists { delete(connections, conn) } // Remove the connection from our waiting map if present. // This must happen before session removal, as otherwise // auth could complete between the session check and this one. waitingLock.Lock() if conn.waitingAuth != nil { waitingConn := waiting[conn.waitingAuth.requestId] if waitingConn == conn { delete(waiting, conn.waitingAuth.requestId) } } waitingLock.Unlock() // Remove the connection from our sessions map if present. // It will not be present if we are degraded, // or have replaced this connection. sessionsLock.Lock() if conn.session != 0 { sessionConn := sessions[conn.session] if sessionConn == conn { delete(sessions, conn.session) // Send a change detaching this node // from that session. idStr := strconv.FormatUint(uint64(config.Id()), 10) ourAttachStr := "attach " + idStr chset := make([]store.Change, 1) chset[0].TargetEntity = conn.session chset[0].Key = ourAttachStr chset[0].Value = "" req := makeRequest(chset) go chrequest.Request(req) } } sessionsLock.Unlock() connectionsLock.Unlock() }() for { select { case msg, ok := <-conn.conn.Received: if !ok { return } switch *msg.MsgType { case 2: handleAuth(conn, msg.Content) case 3: handleFollowUsername(conn, msg.Content) case 4: handleFollowUser(conn, msg.Content) case 5: handleStopFollowingUser(conn, msg.Content) case 6: handleSend(conn, msg.Content) default: conn.conn.Close() } case userMsg := <-conn.deliver: // Get the sender's user ID. store.StartTransaction() attachedTo := store.AllAttachedTo(userMsg.Sender) store.EndTransaction() // We don't know who the sender is. Drop message. if len(attachedTo) == 0 { break } // Deliver any user messages given to us. var deliverMsg cliproto_down.Received deliverMsg.SenderUserId = new(uint64) deliverMsg.SenderSessionId = new(uint64) deliverMsg.Tag = new(string) deliverMsg.Content = new(string) *deliverMsg.SenderUserId = attachedTo[0] *deliverMsg.SenderSessionId = userMsg.Sender *deliverMsg.Tag = userMsg.Tag *deliverMsg.Content = userMsg.Content conn.conn.SendProto(10, &deliverMsg) } } }
func process() { for { select { // Try to make any needed outgoing connections. case <-tryOutgoingCh: store.StartTransaction() connectionsLock.Lock() coreNodes := config.CoreNodes() if !store.Degraded() { if config.IsCore() { for _, node := range coreNodes { if node == config.Id() { continue } if connections[node] == nil { go tryOutgoing(node) } } } else { // Settle for less than 2 core nodes, // if there *aren't* 2 core nodes. targetConns := 2 if targetConns > len(coreNodes) { targetConns = len(coreNodes) } newOutgoing := targetConns - len(connections) used := -1 for newOutgoing > 0 { r := processRand.Intn( len(coreNodes)) // Don't do the same node // twice. if r == used { continue } node := coreNodes[r] if connections[node] == nil { used = r go tryOutgoing(node) newOutgoing-- } } } } else { if len(connections) == 0 { r := processRand.Intn(len(coreNodes)) node := coreNodes[r] if connections[node] == nil { go tryOutgoing(node) } } } connectionsLock.Unlock() store.EndTransaction() // After a random time, check again. randDur := time.Duration(processRand.Intn(19)+1) * time.Second tryOutgoingCh = time.NewTimer(randDur).C // New received connection. case conn := <-receivedConnCh: conn.offerTimers = make(map[uint64]*time.Timer) store.StartTransaction() connectionsLock.Lock() // If we are degraded, reject the connection, // unless we have no other connection, // and it is outbound. if store.Degraded() && !(len(connections) == 0 && conn.outgoing) { conn.lock.Lock() conn.Close() conn.lock.Unlock() connectionsLock.Unlock() store.EndTransaction() break } // If we have an existing connection to this node, // close it. if connections[conn.node] != nil { other := connections[conn.node] other.lock.Lock() other.Close() if other.firstUnappliedTimer != nil { other.firstUnappliedTimer.Stop() } other.lock.Unlock() } // Add to our connections. connections[conn.node] = conn // Send initial position and leader messages. posMsg := new(fproto.Position) posMsg.FirstUnapplied = new(uint64) posMsg.Degraded = new(bool) *posMsg.FirstUnapplied = store.InstructionFirstUnapplied() *posMsg.Degraded = store.Degraded() conn.conn.SendProto(2, posMsg) proposal, leader := store.Proposal() leaderMsg := new(fproto.Leader) leaderMsg.Proposal = new(uint64) leaderMsg.Leader = new(uint32) *leaderMsg.Proposal = proposal *leaderMsg.Leader = uint32(leader) conn.conn.SendProto(11, leaderMsg) connectionsLock.Unlock() store.EndTransaction() // Start timer for sending first unapplied // instruction updates. if config.IsCore() && conn.node <= 0x2000 { conn.lock.Lock() conn.firstUnappliedTimer = time.AfterFunc( firstUnappliedTimerDuration, func() { conn.firstUnappliedTimeout() }) conn.lock.Unlock() } // Start handling received messages from the connection. go handleConn(conn) // Terminated connection. case conn := <-terminatedConnCh: connectionsLock.Lock() if connections[conn.node] == conn { delete(connections, conn.node) conn.lock.Lock() conn.closed = true if conn.firstUnappliedTimer != nil { conn.firstUnappliedTimer.Stop() } conn.lock.Unlock() } connectionsLock.Unlock() } } }
// Must be called from the processing goroutine, inside a transaction. func addPromise(node uint16, msg *coproto.Promise) { receivedPromises[node] = msg // If we have promises from a majority of core nodes, // become leader. if len(receivedPromises) > len(config.CoreNodes())/2 { log.Print("core/consensus: became leader") stopLeaderTimeout() amLeader = true proposal, leader := store.Proposal() // Find a slot number above all those in promise messages, // and above our first unapplied. firstUnapplied := store.InstructionFirstUnapplied() limit := firstUnapplied for _, msg := range receivedPromises { for _, accepted := range msg.Accepted { if *accepted.Slot >= limit { limit = *accepted.Slot + 1 } } } // Start our next slot after the limit. nextProposalSlot = limit // For all slots between this and our first unapplied, // submit a previously accepted instruction unless we // know an instruction was already chosen. // Fills these slots with proposals. // This is O(n^2) in the number of instructions between our // first unapplied and limit. // TODO: Improve worst-case complexity. start := store.InstructionStart() slots := store.InstructionSlots() for i := firstUnapplied; i < limit; i++ { // If we already have a chosen instruction, skip. rel := int(i - start) if len(slots[rel]) == 1 && slots[rel][0].IsChosen() { continue } // Find the previously accepted instruction // accepted with the highest proposal number. var bestInst *coproto.Instruction var bp uint64 // Best proposal var bl uint16 // Best leader for _, msg := range receivedPromises { for _, accepted := range msg.Accepted { if *accepted.Slot != i { continue } if bestInst == nil { bestInst = accepted bp = *accepted.Proposal bl = uint16(*accepted.Leader) continue } // TODO: This indent is just absurd. p := *accepted.Proposal l := uint16(*accepted.Leader) if store.CompareProposals(p, l, bp, bl) { bestInst = accepted bp = *accepted.Proposal bl = uint16(*accepted.Leader) } } } // If we didn't find an instruction, make an empty one. if bestInst == nil { empty := new(coproto.ChangeRequest) empty.RequestEntity = new(uint64) empty.RequestNode = new(uint32) *empty.RequestEntity = uint64(config.Id()) *empty.RequestNode = uint32(config.Id()) bestInst := new(coproto.Instruction) bestInst.Slot = new(uint64) *bestInst.Slot = i bestInst.Request = empty } // Add proposal timeout. req := makeExtChangeRequest(bestInst.Request) addProposalTimeout(i, req) // Send proposal. bestInst.Proposal = new(uint64) bestInst.Leader = new(uint32) *bestInst.Proposal = proposal *bestInst.Leader = uint32(leader) sendProposal(bestInst) } // Discard received promise messages. receivedPromises = nil // Make an instruction proposal for each waiting change. for _, req := range waitingRequests { slot := nextProposalSlot nextProposalSlot++ addProposalTimeout(slot, req) inst := makeInst(slot, proposal, leader, req) sendProposal(inst) } // Clear waiting changes. waitingRequests = nil } }
// Top-level function of the processing goroutine. func process() { connections = make(map[uint16][]*connect.BaseConn) // On startup, make an outgoing connection attempt to all other // core nodes, before continuing. for _, node := range config.CoreNodes() { if node == config.Id() { continue } conn, err := connect.Dial( connect.CONSENSUS_PROTOCOL, node) if err != nil { // Can't reach the other node. continue } log.Print("core/consensus: made outgoing connection to ", node) connections[node] = append(connections[node], conn) go handleConn(node, conn) } // Retry connections once per config.CHANGE_TIMEOUT_PERIOD. // Largely arbitrary. reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD) for { select { // Connection retry tick. // We should try to make an outgoing connection to any node // that we do not have at least one connection to. // We need to make these asynchronously, because connections // are slow. case <-reconnectTicker: for _, node := range config.CoreNodes() { if node == config.Id() { continue } if len(connections[node]) > 0 { continue } go outgoingConn(node) } // New change request, for us to propose as leader. case req := <-newChangeCh: store.StartTransaction() if !amLeader && receivedPromises != nil { waitingRequests = append(waitingRequests, req) } else if !amLeader { waitingRequests = append(waitingRequests, req) // Start attempting to be leader. m := make(map[uint16]*coproto.Promise) receivedPromises = m proposal, _ := store.Proposal() proposal++ store.SetProposal(proposal, config.Id()) firstUn := store.InstructionFirstUnapplied() // Send prepare messages to all other nodes. var prepare coproto.Prepare prepare.Proposal = new(uint64) prepare.FirstUnapplied = new(uint64) *prepare.Proposal = proposal *prepare.FirstUnapplied = firstUn for _, node := range config.CoreNodes() { if node == config.Id() { continue } if len(connections[node]) != 0 { c := connections[node][0] c.SendProto(2, &prepare) } } // Behave as if we got a promise message // from ourselves. var promise coproto.Promise promise.Proposal = prepare.Proposal promise.PrevProposal = promise.Proposal promise.Leader = new(uint32) *promise.Leader = uint32(config.Id()) promise.PrevLeader = promise.Leader addPromise(config.Id(), &promise) } else { newSlot := nextProposalSlot nextProposalSlot++ addProposalTimeout(newSlot, req) proposal, leader := store.Proposal() inst := makeInst(newSlot, proposal, leader, req) sendProposal(inst) } store.EndTransaction() // Leadership attempt timed out. case timedOutProposal := <-leaderTimeoutCh: store.StartTransaction() proposal, leader := store.Proposal() // If the proposal has changed since that // leadership attempt, ignore it. if leader != config.Id() { return } if proposal != timedOutProposal { return } // If we successfully became leader, ignore it. if amLeader { return } // Otherwise, stop our attempt to become leader. stopBeingLeader() store.EndTransaction() // Proposal timed out. case timeout := <-proposalTimeoutCh: store.StartTransaction() // If this timeout was not canceled, a proposal failed. // We stop being leader. if proposalTimeouts[timeout.slot] == timeout { stopBeingLeader() } store.EndTransaction() // New received connection. case receivedConn := <-receivedConnCh: node := receivedConn.node conn := receivedConn.conn connections[node] = append(connections[node], conn) // Received message. case recvMsg := <-receivedMsgCh: node := recvMsg.node conn := recvMsg.conn msg := recvMsg.msg switch *msg.MsgType { case 2: processPrepare(node, conn, msg.Content) case 3: processPromise(node, conn, msg.Content) case 4: processAccept(node, conn, msg.Content) case 5: processAccepted(node, conn, msg.Content) case 6: processNack(node, conn, msg.Content) default: // Unknown message. conn.Close() } // Terminate received connection. case terminatedConn := <-terminatedConnCh: node := terminatedConn.node conn := terminatedConn.conn for i, other := range connections[node] { if other != conn { continue } conns := connections[node] conns = append(conns[:i], conns[i+1:]...) connections[node] = conns break } } } }
// Must be called from the processing goroutine. func processPrepare(node uint16, conn *connect.BaseConn, content []byte) { var msg coproto.Prepare if err := proto.Unmarshal(content, &msg); err != nil { conn.Close() return } store.StartTransaction() defer store.EndTransaction() newProposal, newLeader := *msg.Proposal, node proposal, leader := store.Proposal() if store.CompareProposals(newProposal, newLeader, proposal, leader) { log.Print("core/consensus: sending promise to ", newLeader) // Create a promise message to send back. var promise coproto.Promise promise.Proposal = new(uint64) promise.Leader = new(uint32) promise.PrevProposal = new(uint64) promise.PrevLeader = new(uint32) *promise.Proposal = newProposal *promise.Leader = uint32(newLeader) *promise.PrevProposal = proposal *promise.PrevLeader = uint32(leader) // Add all the instructions we've previously accepted or chosen. slots := store.InstructionSlots() theirFirstUnapplied := *msg.FirstUnapplied ourStart := store.InstructionStart() relativeSlot := int(theirFirstUnapplied - ourStart) if relativeSlot < 0 { relativeSlot = 0 } var accepted []*coproto.Instruction for ; relativeSlot < len(slots); relativeSlot++ { slot := slots[relativeSlot] slotNum := ourStart + uint64(relativeSlot) for i, _ := range slot { if slot[i].IsChosen() { appendInst(&accepted, slotNum, slot[i]) break } weAccepted := false for _, node := range slot[i].Accepted() { if node == config.Id() { weAccepted = true break } } if weAccepted { appendInst(&accepted, slotNum, slot[i]) break } } } // Send promise message. conn.SendProto(3, &promise) // Accept the other node as our new leader. store.SetProposal(newProposal, newLeader) } else { var nack coproto.Nack nack.Proposal = new(uint64) nack.Leader = new(uint32) *nack.Proposal = proposal *nack.Leader = uint32(leader) conn.SendProto(6, &nack) } }
func handleProposalChange() { _, leader := store.Proposal() if amLeader && leader != config.Id() { stopBeingLeader() } }
// Handle a received change forward. Decides which node is responsible for it. // Must only be called from the processing goroutine. func processForward(forward *chproto.ChangeForward) { // If we are already trying to forward a change forward message with // the same requesting node and request ID, discard this message. if _, exists := getForwardTimeout(uint16(*forward.Request.RequestNode), *forward.Request.RequestId); exists { return } // Everything else in this function runs in a transaction. // We are read-only. store.StartTransaction() defer store.EndTransaction() // If this is a core node and this node stopped being leader less than // a Change Timeout Period ago, always add us to the ignore list. if config.IsCore() && !isIgnored(forward, config.Id()) { diff := time.Now().Sub(store.StoppedLeading()) if diff < config.CHANGE_TIMEOUT_PERIOD { forward.Ignores = append(forward.Ignores, uint32(config.Id())) } } // If all core node IDs are in the forward's ignore list, discard it. if len(forward.Ignores) == len(config.CoreNodes()) { log.Print("shared/chrequest: dropped msg due to full ignores") return } // Otherwise, choose a potential leader node. // This is O(n^2) in the number of core nodes, // but we don't expect to have many. chosenNode := uint16(0) _, leader := store.Proposal() if leader != 0 && !isIgnored(forward, leader) { chosenNode = leader } else { for _, node := range config.CoreNodes() { if !isIgnored(forward, node) { chosenNode = node break } } } if chosenNode == 0 { // Shouldn't happen. log.Print("shared/chrequest: bug, " + "couldn't find candidate leader node") return } // If we are the selected leader, construct an external change request, // and send it on our change request channel. if chosenNode == config.Id() { intRequest := forward.Request chrequest := new(store.ChangeRequest) chrequest.RequestEntity = *intRequest.RequestEntity chrequest.RequestNode = uint16(*intRequest.RequestNode) chrequest.RequestId = *intRequest.RequestId chrequest.Changeset = make([]store.Change, len(intRequest.Changeset)) for i, ch := range intRequest.Changeset { chrequest.Changeset[i].TargetEntity = *ch.TargetEntity chrequest.Changeset[i].Key = *ch.Key chrequest.Changeset[i].Value = *ch.Value } for _, cb := range changeCallbacks { cb(chrequest) } return } // Otherwise, we send it on to the selected leader, // add the selected leader to the ignore list, // and set a timeout to retry. sendForward(chosenNode, forward) forward.Ignores = append(forward.Ignores, uint32(chosenNode)) addForwardTimeout(forward) }
func main() { haltChan = make(chan struct{}) // Define and parse flags. id := flag.Uint("id", 0, "Set the node ID of this unanimity instance.") memprof = flag.Uint("memprof", 0, "Generate a memory profile and "+ "halt after the given instruction slot is applied.") cpuprof = flag.Uint("cpuprof", 0, "Generate a CPU profile and halt "+ "after the given instruction slot is applied.") flag.Parse() // Validate flags. if *id == 0 || *id > 0xFFFF { log.Fatal("Invalid node ID specified.") } // If both memprof and cpuprof are set, they need to be the same. if *memprof != 0 && *cpuprof != 0 && *memprof != *cpuprof { log.Fatal("If both memprof and cpuprof are set they must " + "match.") } // If either memprof or cpuprof are set, hook up the callback // responsible for terminating the node at the right time. if *memprof != 0 || *cpuprof != 0 { store.AddAppliedCallback(handleProfileTime) // If generating a CPU profile, start it now. if *cpuprof != 0 { f, err := os.Create("cpuprofile.prof") if err != nil { log.Fatal(err) } pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } } // Load configuration from config file. loadConfig() config.SetId(uint16(*id)) // Load our TLS certificate. idStr := strconv.FormatUint(uint64(config.Id()), 10) cert, err := tls.LoadX509KeyPair(idStr+".crt", idStr+".key") if err != nil { log.Fatalf("error loading our TLS certificate: %s", err) } config.SetCertificate(&cert) fmt.Printf("Loaded configuration, our ID is %d.\n", config.Id()) if config.IsCore() { core.Startup() } else { client.Startup() } <-haltChan // If we're to generate a memory profile before terminating, do so. if *memprof != 0 { f, err := os.Create("memprofile.mprof") if err != nil { log.Fatal(err) } defer f.Close() pprof.WriteHeapProfile(f) } }
// Blocks listening for connections of that given protocol type, // and calls the specified handler when one is received, // passing the node ID of the connecting node, or 0 for the Client Protocol. func Listen(protocol int, handler func(id uint16, b *BaseConn)) { ip := config.NodeIP(config.Id()) ipStr := ip.String() port := getProtocolPort(protocol) portStr := strconv.FormatInt(int64(port), 10) tlsConfig := new(tls.Config) tlsConfig.Certificates = []tls.Certificate{*config.Certificate()} if protocol != CLIENT_PROTOCOL { tlsConfig.ClientAuth = tls.RequireAnyClientCert } listener, err := tls.Listen("tcp", ipStr+":"+portStr, tlsConfig) if err != nil { log.Fatal(err) } for { conn, err := listener.Accept() if err != nil { log.Fatal(err) } tlsConn := conn.(*tls.Conn) err = tlsConn.Handshake() if err != nil { tlsConn.Close() continue } if protocol != CLIENT_PROTOCOL { // Check this connecting node has authenticated. state := tlsConn.ConnectionState() if len(state.PeerCertificates) == 0 { tlsConn.Close() continue } cert := state.PeerCertificates[0] // Identify which node they authenticated as. matched := false for _, node := range config.Nodes() { var verifyOpts x509.VerifyOptions verifyOpts.Intermediates = new(x509.CertPool) verifyOpts.Roots = config.NodeCertPool(node) chains, err := cert.Verify(verifyOpts) if err != nil { continue } // Matched the node. Start the handler. if len(chains) > 0 { matched = true go handler(node, newBaseConn(tlsConn)) break } } // No matching node found. Close the connection. if !matched { tlsConn.Close() } } else { // We don't authenticate clients. // Just run the handler. handler(0, newBaseConn(tlsConn)) } } }
// Sets this instruction value as chosen, clearing all others in the same slot. // Causes it to be automatically applied to state once all previous instruction // slots have a chosen value. func (i *InstructionValue) Choose() { log.Print("shared/store: choosing instruction in slot ", i.slot) relativeSlot := i.slot - start // Drop all other instruction values in the same slot. if len(slots[relativeSlot]) > 1 { slots[relativeSlot] = []*InstructionValue{i} } // Set the value as chosen. i.chosen = true i.chosenTime = time.Now() i.acceptedBy = nil // Call instruction chosen callbacks. for _, cb := range chosenCallbacks { cb(i.slot) } // Apply instructions as far as possible. tryApply() // Discard no longer needed instructions, from the start. newStart := start for { relative := newStart - start // If we've not applied the instruction yet, // we still need it. if firstUnapplied == newStart { break } // If the instruction wasn't chosen long enough ago, // we can't discard it. minDelay := time.Duration(4) * config.CHANGE_TIMEOUT_PERIOD timeChosen := slots[relative][0].TimeChosen() if time.Now().Sub(timeChosen) < minDelay { break } // If this is a core node and the majority of core nodes aren't // past this point, we have to keep it. if config.IsCore() { past := 1 // We are past it. coreNodes := config.CoreNodes() for _, node := range coreNodes { if node == config.Id() { continue } if NodeFirstUnapplied(node) > newStart { past++ } } if past <= len(coreNodes)/2 { break } } newStart++ } // If we can discard, do it by reslicing the slots. // This doesn't result in an immediate free, // but will when we next expand the slots. // This avoids extra performance costs for reallocating them now. if newStart != start { log.Print("Discarding instructions between ", newStart-1, " and ", start) slots = slots[newStart-start:] start = newStart } }
func handleApplied(slot uint64, idemChanges []store.Change) { relativeSlot := int(slot - store.InstructionStart()) slots := store.InstructionSlots() requestNode := slots[relativeSlot][0].ChangeRequest().RequestNode requestId := slots[relativeSlot][0].ChangeRequest().RequestId sessionsLock.Lock() defer sessionsLock.Unlock() waitingLock.Lock() defer waitingLock.Unlock() // If there is a connection which has this as its waiting request, // have it complete authentication. if requestNode == config.Id() { waitingConn := waiting[requestId] if waitingConn != nil { handleAuthComplete(waitingConn, idemChanges) } } ourAttachStr := "attach " + strconv.FormatUint(uint64(config.Id()), 10) for i := range idemChanges { key := idemChanges[i].Key value := idemChanges[i].Value if strings.HasPrefix(key, "attach ") { // If we were just detached from a session, // drop any connections associated with it, // and if it was in our attach timeouts, remove it. if key == ourAttachStr && value == "" { sessionId := idemChanges[i].TargetEntity if conn := sessions[sessionId]; conn != nil { log.Print("client/listener: " + "detached from existing " + "session") conn.conn.Close() } if orphanAttachTimeouts[sessionId] != nil { orphanAttachTimeouts[sessionId].Stop() delete(orphanAttachTimeouts, sessionId) } } // If we were just attached to a session, // but have no connection associated with that session, // add an attach timeout. // and if it was in our attach timeouts, remove it. if key == ourAttachStr && value != "" { sessionId := idemChanges[i].TargetEntity if sessions[sessionId] == nil { startOrphanTimeout(sessionId) } } // Handle a session attaching/detaching from a user. entityId := idemChanges[i].TargetEntity entity := store.GetEntity(entityId) if entity != nil && entity.Value("kind") == "user" { // Tell following connections when a session // attaches or detaches from a followed user. // TODO: Expensive loop, // should maintain index to avoid this. for conn, _ := range connections { following := false for _, fw := range conn.following { if fw == entityId { following = true break } } if !following { continue } sendUserChange(conn, entityId, key, value) } } } } // If we've created a nameless user, start a timer to delete it. for i := range idemChanges { if idemChanges[i].Key != "kind" || idemChanges[i].Value != "user" { continue } newUserId := idemChanges[i].TargetEntity newUser := store.GetEntity(newUserId) if newUser != nil && newUser.Value("name username") == "" { startNamelessTimeout(newUserId) } } }
// Can only be called from the handling goroutine for conn. func handleAuth(conn *userConn, content []byte) { var msg cliproto_up.Authenticate if err := proto.Unmarshal(content, &msg); err != nil { conn.conn.Close() return } // Try to get information, then release locks and hash the password. // If the situation changes we may have to hash it again anyway, // but scrypt hashing is extremely expensive and we want to try to // do this without holding our locks in the vast majority of cases. var tryPassGenerated bool var trySaltGenerated bool var tryPass, trySalt, tryKey string if *msg.Password != "" { tryPass = *msg.Password store.StartTransaction() userId := store.NameLookup("user", "name username", *msg.Username) if userId != 0 { user := store.GetEntity(userId) if user != nil { trySalt = user.Value("auth salt") } } store.EndTransaction() if trySalt == "" { trySaltGenerated = true var err error trySalt, tryKey, err = genRandomSalt(conn, []byte(tryPass)) if err != nil { return } } else { var err error tryPass, err = genKey(conn, []byte(tryPass), []byte(trySalt)) if err != nil { return } } } else { tryPassGenerated = true trySaltGenerated = true var err error tryPass, trySalt, tryKey, err = genRandomPass(conn) if err != nil { return } } // TODO: Validate username and password against constraints. // We hold this through quite a lot of logic. // Would be good to be locked less. store.StartTransaction() defer store.EndTransaction() sessionsLock.Lock() defer sessionsLock.Unlock() waitingLock.Lock() defer waitingLock.Unlock() if conn.session != 0 || conn.waitingAuth != nil { conn.conn.Close() return } userId := store.NameLookup("user", "name username", *msg.Username) if userId != 0 { if *msg.Password == "" { sendAuthFail(conn, "Invalid Password") return } // The user already exists. user := store.GetEntity(userId) // Try to authenticate them to it. var key string // If their salt and password matches our attempt above, // we can just take that key. salt := user.Value("auth salt") if trySalt == salt && tryPass == *msg.Password { key = tryKey } else { saltBytes := []byte(user.Value("auth salt")) passBytes := []byte(*msg.Password) var err error key, err = genKey(conn, passBytes, saltBytes) if err != nil { return } } if user.Value("auth password") != string(key) { sendAuthFail(conn, "Invalid Password") return } // It's the real user. if *msg.SessionId != 0 { // They are attaching to an existing session. // Check it exists and is attached to this user. strSessionId := strconv.FormatUint(*msg.SessionId, 10) if user.Value("attach "+strSessionId) == "" { sendAuthFail(conn, "Invalid Session") return } // The session does exist. conn.session = *msg.SessionId // If this node is already attached to // the session, drop the other connection. if sessions[conn.session] != nil { sessions[conn.session].conn.Close() } else { // Create change attaching this node ID // to the session. id := config.Id() idStr := strconv.FormatUint(uint64(id), 10) chset := make([]store.Change, 1) chset[0].TargetEntity = conn.session chset[0].Key = "attach " + idStr chset[0].Value = "true" req := makeRequest(chset) go chrequest.Request(req) } // Put us in the sessions map. sessions[conn.session] = conn // Tell the client they authenticated successfully. sendAuthSuccess(conn, "") } else { // They are creating a new session. req := makeNewSessionRequest(userId) go chrequest.Request(req) // Stuff details in waiting auth. conn.waitingAuth = new(authData) conn.waitingAuth.msg = msg conn.waitingAuth.requestId = req.RequestId waiting[conn.waitingAuth.requestId] = conn } } else { // The user does not already exist. // Check they weren't trying to attach to a session. if *msg.SessionId != 0 { sendAuthFail(conn, "User Does Not Exist") return } // We're creating a new user. newUser := *msg.Username newPass := *msg.Password if !strings.HasPrefix(newUser, "Guest-") { // We're creating a new non-guest user. // Make sure they have a password. if newPass == "" { sendAuthFail(conn, "No Password") return } var salt string var hash string if tryPass == newPass && trySaltGenerated { salt = trySalt hash = tryKey } else { passBytes := []byte(newPass) var err error salt, hash, err = genRandomSalt(conn, passBytes) if err != nil { return } } // Create the new user. req := makeNewUserRequest(newUser, hash, salt, false) go chrequest.Request(req) // Stuff details in waiting auth. conn.waitingAuth = new(authData) conn.waitingAuth.msg = msg conn.waitingAuth.requestId = req.RequestId waiting[conn.waitingAuth.requestId] = conn return } // We're creating a new guest user. // Guests get automatic passwords, and can't set them. if newPass != "" { sendAuthFail(conn, "Cannot Set Password For Guest User") return } var hash string var salt string if tryPassGenerated && trySaltGenerated { newPass = tryPass salt = trySalt hash = tryKey } else { var err error newPass, salt, hash, err = genRandomPass(conn) if err != nil { return } } waitingLock.Lock() // Create the new user. req := makeNewUserRequest(newUser, hash, salt, true) go chrequest.Request(req) // Stuff details in waiting auth. conn.waitingAuth = new(authData) conn.waitingAuth.msg = msg conn.waitingAuth.requestId = req.RequestId waiting[conn.waitingAuth.requestId] = conn waitingLock.Unlock() return } }
// If the leader ID matches our ID, // updates our "last stopped leading" time to now. // Call when, whether or not we've a new leader, // we consider ourselves to have stopped being the leader. func StopLeading() { if intLeader == config.Id() { lastLeader = time.Now() } }
func process() { // Try to make an outgoing connection to all other client nodes, // if we're not in a degraded state. store.StartTransaction() if !store.Degraded() { for _, node := range config.ClientNodes() { if node == config.Id() { continue } conn, err := connect.Dial(connect.RELAY_PROTOCOL, node) if err != nil { // No luck connecting. continue } connections[node] = append(connections[node], conn) go handleConn(node, conn) } } store.EndTransaction() // Retry connections once per config.CHANGE_TIMEOUT_PERIOD // Largely arbitrary. reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD) for { select { // Connection retry tick. // If not degraded, try to make an outgoing connection to any // client node that we do not have at least one connection to. case <-reconnectTicker: store.StartTransaction() // Do not attempt to make connections while degraded. if store.Degraded() { store.EndTransaction() break } for _, node := range config.ClientNodes() { if node == config.Id() { continue } if len(connections[node]) > 0 { continue } conn, err := connect.Dial( connect.RELAY_PROTOCOL, node) if err != nil { // No luck connecting. continue } connections[node] = append(connections[node], conn) go handleConn(node, conn) } store.EndTransaction() // New received connection. case receivedConn := <-receivedConnCh: node := receivedConn.node conn := receivedConn.conn store.StartTransaction() // If we are degraded, reject the connection. if store.Degraded() { conn.Close() store.EndTransaction() break } // Add to our connections. connections[node] = append(connections[node], conn) store.EndTransaction() // Terminated connection. case receivedConn := <-terminatedConnCh: node := receivedConn.node conn := receivedConn.conn store.StartTransaction() // Remove this connection from our connection list. index := -1 for i, _ := range connections[node] { if conn == connections[node][i] { index = i break } } if index != -1 { connections[node] = append(connections[node][:index], connections[node][index+1:]...) } store.EndTransaction() } } }