// CraftAndSubmitFullFault is called from the Negotiate goroutine
// (which fires once every 5 seconds on each server); most of the time
// these are "incomplete" FullFault messages which serve as status pings
// for the negotiation in progress
func CraftAndSubmitFullFault(pl *ProcessList, faultID [32]byte) *messages.FullServerFault {
faultState := pl.GetFaultState(faultID)
fc := faultState.FaultCore
sf := messages.NewServerFault(fc.ServerID, fc.AuditServerID, int(fc.VMIndex), fc.DBHeight, fc.Height, pl.System.Height, fc.Timestamp)
var listOfSigs []interfaces.IFullSignature
for _, sig := range faultState.VoteMap {
listOfSigs = append(listOfSigs, sig)
}
var pff *messages.FullServerFault
if pl.System.Height > 0 {
pff = pl.System.List[pl.System.Height-1].(*messages.FullServerFault)
}
fullFault := messages.NewFullServerFault(pff, sf, listOfSigs, pl.System.Height)
//adminBlockEntryForFault := fullFault.ToAdminBlockEntry()
//pl.State.LeaderPL.AdminBlock.AddServerFault(adminBlockEntryForFault)
if fullFault != nil {
fullFault.Sign(pl.State.serverPrivKey)
fullFault.SendOut(pl.State, fullFault)
fullFault.FollowerExecute(pl.State)
}
return fullFault
}
// CraftAndSubmitFault is how we issue ServerFault messages when we have an
// open choice of which Audit Server we want to vote for (i.e. when we're not
// just matching someone else's existing Fault message vote)
func CraftAndSubmitFault(pl *ProcessList, vmIndex int, height int) {
// TODO: if I am the Leader being faulted, I should respond by sending out
// a MissingMsgResponse to everyone for the msg I'm being faulted for
// Only consider Online Audit servers as candidates for promotion (this
// allows us to cycle through Audits on successive calls to CraftAndSubmitFault,
// so that we make sure to (eventually) find one that is ready and able to
// accept the promotion)
auditServerList := pl.State.GetOnlineAuditServers(pl.DBHeight)
if len(auditServerList) > 0 {
// Nominate the top candidate from the list of Online Audit Servers
replacementServer := auditServerList[0]
leaderMin := pl.State.CurrentMinute
faultedFed := pl.ServerMap[leaderMin][vmIndex]
pl.FedServers[faultedFed].SetOnline(false)
faultedFedID := pl.FedServers[faultedFed].GetChainID()
// Create and send ServerFault (vote) message
sf := messages.NewServerFault(faultedFedID, replacementServer.GetChainID(), vmIndex, pl.DBHeight, uint32(height), pl.System.Height, pl.State.GetTimestamp())
if sf != nil {
sf.Sign(pl.State.serverPrivKey)
//pl.State.NetworkOutMsgQueue() <- sf
pl.State.InMsgQueue() <- sf
fm := pl.GetFaultState(sf.GetCoreHash().Fixed())
if !fm.IsNil() {
// If we already have a FaultState saved to our ProcessList's
// FaultMap, we update its "LastMatch" value to the current time
// (LastMatch is merely a throttling mechanism)
fm.LastMatch = time.Now().Unix()
pl.AddFaultState(sf.GetCoreHash().Fixed(), fm)
}
}
} else {
// If we don't see any Audit servers as Online, we reset all of
// them to an Online state and start the cycle anew
for _, aud := range pl.AuditServers {
aud.SetOnline(true)
}
}
}
// When we execute a FullFault message, it could be complete (includes all
// necessary signatures + pledge) or incomplete, in which case it is just
// a negotiation ping
func (s *State) FollowerExecuteFullFault(m interfaces.IMsg) {
fullFault, _ := m.(*messages.FullServerFault)
pl := s.ProcessLists.Get(fullFault.DBHeight)
if fullFault.ServerID.IsZero() || fullFault.AuditServerID.IsZero() {
return
}
auditServerList := s.GetAuditServers(fullFault.DBHeight)
var theAuditReplacement interfaces.IFctServer
for _, auditServer := range auditServerList {
if auditServer.GetChainID().IsSameAs(fullFault.AuditServerID) {
theAuditReplacement = auditServer
}
}
if theAuditReplacement == nil {
// If we don't have any Audit Servers in our Authority set
// that match the nominated Audit Server in the FullFault,
// we can't really do anything useful with it
return
}
vm := pl.VMs[int(fullFault.VMIndex)]
rHt := vm.Height
ffHt := int(fullFault.Height)
if false && rHt > ffHt {
fmt.Printf("dddd %20s VM[%d] height %d Full Fault ht: %d \n", s.FactomNodeName, fullFault.VMIndex, rHt, ffHt)
vm.Height = ffHt
vm.List = vm.List[:ffHt] // Nuke all the extra messages that might annoy us.
}
if int(fullFault.SystemHeight) == pl.System.Height {
if fullFault.HasEnoughSigs(s) && s.pledgedByAudit(fullFault) {
// COMPLETE
pl.AddToSystemList(fullFault)
} else {
mightMatch := false
willUpdate := false
if s.IdentityChainID.IsSameAs(fullFault.AuditServerID) {
// We are the nominated auditServer
mightMatch = true
} else {
if vm.whenFaulted != 0 {
//I AGREE
currentTopPriority := TopPriorityFaultState(pl)
var tpts int64
if currentTopPriority.IsNil() {
tpts = 0
} else {
tpts = currentTopPriority.FaultCore.Timestamp.GetTimeSeconds()
}
ffts := fullFault.Timestamp.GetTimeSeconds()
if ffts >= tpts {
//THIS IS TOP PRIORITY
if !currentTopPriority.IsNil() && fullFault.ServerID.IsSameAs(currentTopPriority.FaultCore.ServerID) && ffts > tpts {
//IT IS A RENEWAL
if int(ffts-tpts) < s.FaultTimeout {
//TOO SOON
newVMI := (int(fullFault.VMIndex) + 1) % len(pl.FedServers)
Fault(pl, newVMI, int(fullFault.Height))
} else {
if !currentTopPriority.IsNil() && couldIFullFault(pl, int(currentTopPriority.FaultCore.VMIndex)) {
//I COULD FAULT BUT HE HASN'T
newVMI := (int(fullFault.VMIndex) + 1) % len(pl.FedServers)
Fault(pl, newVMI, int(fullFault.Height))
} else {
willUpdate = true
}
}
} else {
willUpdate = true
}
}
}
}
if willUpdate {
mightMatch = true
s.regularFullFaultExecution(fullFault, pl)
}
if mightMatch {
theFaultState := pl.GetFaultState(fullFault.GetCoreHash().Fixed())
if !theFaultState.MyVoteTallied {
now := time.Now().Unix()
if now-theFaultState.LastMatch > 5 && int(now-s.LastTiebreak) > s.FaultTimeout/2 {
if theFaultState.SigTally(s) >= len(pl.FedServers)-1 {
s.LastTiebreak = now
}
nsf := messages.NewServerFault(fullFault.ServerID, fullFault.AuditServerID, int(fullFault.VMIndex),
fullFault.DBHeight, fullFault.Height, int(fullFault.SystemHeight), fullFault.Timestamp)
s.matchFault(nsf)
}
}
}
}
}
}
// regularFullFaultExecution does the same thing as regularFaultExecution, except
// it will make sure to add every signature from the FullFault to the corresponding
// FaultState's VoteMap
func (s *State) regularFullFaultExecution(sf *messages.FullServerFault, pl *ProcessList) {
coreHash := sf.GetCoreHash().Fixed()
for _, signature := range sf.SignatureList.List {
var issuerID [32]byte
rawIssuerID := signature.GetKey()
for i := 0; i < 32; i++ {
if i < len(rawIssuerID) {
issuerID[i] = rawIssuerID[i]
}
}
faultState := pl.GetFaultState(coreHash)
if !faultState.IsNil() {
// We already have a map entry
} else {
// We don't have a map entry yet; let's create one
fcore := ExtractFaultCore(sf)
faultState = FaultState{FaultCore: fcore, AmINegotiator: false, MyVoteTallied: false, VoteMap: make(map[[32]byte]interfaces.IFullSignature)}
if isMyNegotiation(fcore, pl) {
faultState.AmINegotiator = true
pl.AmINegotiator = true
}
if faultState.VoteMap == nil {
faultState.VoteMap = make(map[[32]byte]interfaces.IFullSignature)
}
pl.AddFaultState(coreHash, faultState)
}
if s.Leader || s.IdentityChainID.IsSameAs(sf.AuditServerID) {
if !faultState.MyVoteTallied {
nsf := messages.NewServerFault(sf.ServerID, sf.AuditServerID, int(sf.VMIndex), sf.DBHeight, sf.Height, int(sf.SystemHeight), sf.Timestamp)
sfbytes, err := nsf.MarshalForSignature()
myAuth, _ := s.GetAuthority(s.IdentityChainID)
if myAuth == nil || err != nil {
return
}
valid, err := myAuth.VerifySignature(sfbytes, signature.GetSignature())
if err == nil && valid {
faultState.MyVoteTallied = true
pl.AddFaultState(coreHash, faultState)
}
}
}
lbytes := sf.GetCoreHash().Bytes()
isPledge := false
auth, _ := s.GetAuthority(sf.AuditServerID)
if auth == nil {
isPledge = false
} else {
valid, err := auth.VerifySignature(lbytes, signature.GetSignature())
if err == nil && valid {
isPledge = true
faultState.PledgeDone = true
pl.AddFaultState(coreHash, faultState)
}
}
sfSigned, err := s.FastVerifyAuthoritySignature(lbytes, signature, sf.DBHeight)
if err == nil && (sfSigned > 0 || (sfSigned == 0 && isPledge)) {
faultState.VoteMap[issuerID] = sf.GetSignature()
pl.AddFaultState(coreHash, faultState)
}
}
faultState := pl.GetFaultState(coreHash)
if !faultState.IsNil() {
if s.Leader || s.IdentityChainID.IsSameAs(sf.AuditServerID) {
if !faultState.MyVoteTallied {
now := time.Now().Unix()
if int(now-s.LastTiebreak) > s.FaultTimeout/2 {
if faultState.SigTally(s) >= len(pl.FedServers)-1 {
s.LastTiebreak = now
}
nsf := messages.NewServerFault(sf.ServerID, sf.AuditServerID, int(sf.VMIndex), sf.DBHeight, sf.Height, int(sf.SystemHeight), sf.Timestamp)
s.matchFault(nsf)
}
}
}
}
}