// CraftAndSubmitFullFault is called from the Negotiate goroutine // (which fires once every 5 seconds on each server); most of the time // these are "incomplete" FullFault messages which serve as status pings // for the negotiation in progress func CraftAndSubmitFullFault(pl *ProcessList, faultID [32]byte) *messages.FullServerFault { faultState := pl.GetFaultState(faultID) fc := faultState.FaultCore sf := messages.NewServerFault(fc.ServerID, fc.AuditServerID, int(fc.VMIndex), fc.DBHeight, fc.Height, pl.System.Height, fc.Timestamp) var listOfSigs []interfaces.IFullSignature for _, sig := range faultState.VoteMap { listOfSigs = append(listOfSigs, sig) } var pff *messages.FullServerFault if pl.System.Height > 0 { pff = pl.System.List[pl.System.Height-1].(*messages.FullServerFault) } fullFault := messages.NewFullServerFault(pff, sf, listOfSigs, pl.System.Height) //adminBlockEntryForFault := fullFault.ToAdminBlockEntry() //pl.State.LeaderPL.AdminBlock.AddServerFault(adminBlockEntryForFault) if fullFault != nil { fullFault.Sign(pl.State.serverPrivKey) fullFault.SendOut(pl.State, fullFault) fullFault.FollowerExecute(pl.State) } return fullFault }
// CraftAndSubmitFault is how we issue ServerFault messages when we have an // open choice of which Audit Server we want to vote for (i.e. when we're not // just matching someone else's existing Fault message vote) func CraftAndSubmitFault(pl *ProcessList, vmIndex int, height int) { // TODO: if I am the Leader being faulted, I should respond by sending out // a MissingMsgResponse to everyone for the msg I'm being faulted for // Only consider Online Audit servers as candidates for promotion (this // allows us to cycle through Audits on successive calls to CraftAndSubmitFault, // so that we make sure to (eventually) find one that is ready and able to // accept the promotion) auditServerList := pl.State.GetOnlineAuditServers(pl.DBHeight) if len(auditServerList) > 0 { // Nominate the top candidate from the list of Online Audit Servers replacementServer := auditServerList[0] leaderMin := pl.State.CurrentMinute faultedFed := pl.ServerMap[leaderMin][vmIndex] pl.FedServers[faultedFed].SetOnline(false) faultedFedID := pl.FedServers[faultedFed].GetChainID() // Create and send ServerFault (vote) message sf := messages.NewServerFault(faultedFedID, replacementServer.GetChainID(), vmIndex, pl.DBHeight, uint32(height), pl.System.Height, pl.State.GetTimestamp()) if sf != nil { sf.Sign(pl.State.serverPrivKey) //pl.State.NetworkOutMsgQueue() <- sf pl.State.InMsgQueue() <- sf fm := pl.GetFaultState(sf.GetCoreHash().Fixed()) if !fm.IsNil() { // If we already have a FaultState saved to our ProcessList's // FaultMap, we update its "LastMatch" value to the current time // (LastMatch is merely a throttling mechanism) fm.LastMatch = time.Now().Unix() pl.AddFaultState(sf.GetCoreHash().Fixed(), fm) } } } else { // If we don't see any Audit servers as Online, we reset all of // them to an Online state and start the cycle anew for _, aud := range pl.AuditServers { aud.SetOnline(true) } } }
// When we execute a FullFault message, it could be complete (includes all // necessary signatures + pledge) or incomplete, in which case it is just // a negotiation ping func (s *State) FollowerExecuteFullFault(m interfaces.IMsg) { fullFault, _ := m.(*messages.FullServerFault) pl := s.ProcessLists.Get(fullFault.DBHeight) if fullFault.ServerID.IsZero() || fullFault.AuditServerID.IsZero() { return } auditServerList := s.GetAuditServers(fullFault.DBHeight) var theAuditReplacement interfaces.IFctServer for _, auditServer := range auditServerList { if auditServer.GetChainID().IsSameAs(fullFault.AuditServerID) { theAuditReplacement = auditServer } } if theAuditReplacement == nil { // If we don't have any Audit Servers in our Authority set // that match the nominated Audit Server in the FullFault, // we can't really do anything useful with it return } vm := pl.VMs[int(fullFault.VMIndex)] rHt := vm.Height ffHt := int(fullFault.Height) if false && rHt > ffHt { fmt.Printf("dddd %20s VM[%d] height %d Full Fault ht: %d \n", s.FactomNodeName, fullFault.VMIndex, rHt, ffHt) vm.Height = ffHt vm.List = vm.List[:ffHt] // Nuke all the extra messages that might annoy us. } if int(fullFault.SystemHeight) == pl.System.Height { if fullFault.HasEnoughSigs(s) && s.pledgedByAudit(fullFault) { // COMPLETE pl.AddToSystemList(fullFault) } else { mightMatch := false willUpdate := false if s.IdentityChainID.IsSameAs(fullFault.AuditServerID) { // We are the nominated auditServer mightMatch = true } else { if vm.whenFaulted != 0 { //I AGREE currentTopPriority := TopPriorityFaultState(pl) var tpts int64 if currentTopPriority.IsNil() { tpts = 0 } else { tpts = currentTopPriority.FaultCore.Timestamp.GetTimeSeconds() } ffts := fullFault.Timestamp.GetTimeSeconds() if ffts >= tpts { //THIS IS TOP PRIORITY if !currentTopPriority.IsNil() && fullFault.ServerID.IsSameAs(currentTopPriority.FaultCore.ServerID) && ffts > tpts { //IT IS A RENEWAL if int(ffts-tpts) < s.FaultTimeout { //TOO SOON newVMI := (int(fullFault.VMIndex) + 1) % len(pl.FedServers) Fault(pl, newVMI, int(fullFault.Height)) } else { if !currentTopPriority.IsNil() && couldIFullFault(pl, int(currentTopPriority.FaultCore.VMIndex)) { //I COULD FAULT BUT HE HASN'T newVMI := (int(fullFault.VMIndex) + 1) % len(pl.FedServers) Fault(pl, newVMI, int(fullFault.Height)) } else { willUpdate = true } } } else { willUpdate = true } } } } if willUpdate { mightMatch = true s.regularFullFaultExecution(fullFault, pl) } if mightMatch { theFaultState := pl.GetFaultState(fullFault.GetCoreHash().Fixed()) if !theFaultState.MyVoteTallied { now := time.Now().Unix() if now-theFaultState.LastMatch > 5 && int(now-s.LastTiebreak) > s.FaultTimeout/2 { if theFaultState.SigTally(s) >= len(pl.FedServers)-1 { s.LastTiebreak = now } nsf := messages.NewServerFault(fullFault.ServerID, fullFault.AuditServerID, int(fullFault.VMIndex), fullFault.DBHeight, fullFault.Height, int(fullFault.SystemHeight), fullFault.Timestamp) s.matchFault(nsf) } } } } } }
// regularFullFaultExecution does the same thing as regularFaultExecution, except // it will make sure to add every signature from the FullFault to the corresponding // FaultState's VoteMap func (s *State) regularFullFaultExecution(sf *messages.FullServerFault, pl *ProcessList) { coreHash := sf.GetCoreHash().Fixed() for _, signature := range sf.SignatureList.List { var issuerID [32]byte rawIssuerID := signature.GetKey() for i := 0; i < 32; i++ { if i < len(rawIssuerID) { issuerID[i] = rawIssuerID[i] } } faultState := pl.GetFaultState(coreHash) if !faultState.IsNil() { // We already have a map entry } else { // We don't have a map entry yet; let's create one fcore := ExtractFaultCore(sf) faultState = FaultState{FaultCore: fcore, AmINegotiator: false, MyVoteTallied: false, VoteMap: make(map[[32]byte]interfaces.IFullSignature)} if isMyNegotiation(fcore, pl) { faultState.AmINegotiator = true pl.AmINegotiator = true } if faultState.VoteMap == nil { faultState.VoteMap = make(map[[32]byte]interfaces.IFullSignature) } pl.AddFaultState(coreHash, faultState) } if s.Leader || s.IdentityChainID.IsSameAs(sf.AuditServerID) { if !faultState.MyVoteTallied { nsf := messages.NewServerFault(sf.ServerID, sf.AuditServerID, int(sf.VMIndex), sf.DBHeight, sf.Height, int(sf.SystemHeight), sf.Timestamp) sfbytes, err := nsf.MarshalForSignature() myAuth, _ := s.GetAuthority(s.IdentityChainID) if myAuth == nil || err != nil { return } valid, err := myAuth.VerifySignature(sfbytes, signature.GetSignature()) if err == nil && valid { faultState.MyVoteTallied = true pl.AddFaultState(coreHash, faultState) } } } lbytes := sf.GetCoreHash().Bytes() isPledge := false auth, _ := s.GetAuthority(sf.AuditServerID) if auth == nil { isPledge = false } else { valid, err := auth.VerifySignature(lbytes, signature.GetSignature()) if err == nil && valid { isPledge = true faultState.PledgeDone = true pl.AddFaultState(coreHash, faultState) } } sfSigned, err := s.FastVerifyAuthoritySignature(lbytes, signature, sf.DBHeight) if err == nil && (sfSigned > 0 || (sfSigned == 0 && isPledge)) { faultState.VoteMap[issuerID] = sf.GetSignature() pl.AddFaultState(coreHash, faultState) } } faultState := pl.GetFaultState(coreHash) if !faultState.IsNil() { if s.Leader || s.IdentityChainID.IsSameAs(sf.AuditServerID) { if !faultState.MyVoteTallied { now := time.Now().Unix() if int(now-s.LastTiebreak) > s.FaultTimeout/2 { if faultState.SigTally(s) >= len(pl.FedServers)-1 { s.LastTiebreak = now } nsf := messages.NewServerFault(sf.ServerID, sf.AuditServerID, int(sf.VMIndex), sf.DBHeight, sf.Height, int(sf.SystemHeight), sf.Timestamp) s.matchFault(nsf) } } } } }