func FaultCheck(pl *ProcessList) {
faultState := TopPriorityFaultState(pl)
if !faultState.IsNil() {
timeElapsed := time.Now().Unix() - faultState.FaultCore.Timestamp.GetTimeSeconds()
if isMyNegotiation(faultState.FaultCore, pl) {
if int(timeElapsed) > pl.State.FaultTimeout+1 {
// Negotiation has timed out; must issue RENEWAL (new negotiation)
if !faultState.PledgeDone {
// Now let's set the Audit Server offline (so we don't just re-nominate them over and over)
auditServerList := pl.State.GetAuditServers(faultState.FaultCore.DBHeight)
var theAuditReplacement interfaces.IFctServer
for _, auditServer := range auditServerList {
if auditServer.GetChainID().IsSameAs(faultState.FaultCore.AuditServerID) {
theAuditReplacement = auditServer
}
}
if theAuditReplacement != nil {
theAuditReplacement.SetOnline(false)
}
}
CraftAndSubmitFault(pl, int(faultState.FaultCore.VMIndex), int(faultState.FaultCore.Height))
} else {
CraftAndSubmitFullFault(pl, faultState.FaultCore.GetHash().Fixed())
}
} else {
if int(timeElapsed) > pl.State.FaultTimeout*2 {
// The negotiation has expired; time to fault negotiator
newVMI := (int(faultState.FaultCore.VMIndex) + 1) % len(pl.FedServers)
Fault(pl, newVMI, int(faultState.FaultCore.Height))
}
}
}
}
// a contains b, returns true
func containsServer(haystack []interfaces.IFctServer, needle interfaces.IFctServer) bool {
for _, hay := range haystack {
if needle.GetChainID().IsSameAs(hay.GetChainID()) {
return true
}
}
return false
}
func (s *State) ProcessFullServerFault(dbheight uint32, msg interfaces.IMsg) (haveReplaced bool) {
// If we are here, this means that the FullFault message is complete
// and we can execute it as such (replacing the faulted Leader with
// the nominated Audit server)
//fmt.Println("FULL FAULT:", s.FactomNodeName, s.GetTimestamp().GetTimeSeconds())
fullFault, _ := msg.(*messages.FullServerFault)
pl := s.ProcessLists.Get(fullFault.DBHeight)
if pl.System.Height < int(fullFault.SystemHeight) {
return false
}
vm := pl.VMs[int(fullFault.VMIndex)]
if fullFault.Height > uint32(vm.Height) {
return false
}
auditServerList := s.GetAuditServers(fullFault.DBHeight)
var theAuditReplacement interfaces.IFctServer
for _, auditServer := range auditServerList {
if auditServer.GetChainID().IsSameAs(fullFault.AuditServerID) {
theAuditReplacement = auditServer
}
}
if theAuditReplacement == nil {
// If we don't have any Audit Servers in our Authority set
// that match the nominated Audit Server in the FullFault,
// we can't really do anything useful with it
return
}
pl.FaultedVMIndex = int(fullFault.VMIndex)
if fullFault.HasEnoughSigs(s) {
if s.pledgedByAudit(fullFault) {
// If we are here, this means that the FullFault message is complete
// and we can execute it as such (replacing the faulted Leader with
// the nominated Audit server)
rHt := vm.Height
ffHt := int(fullFault.Height)
if rHt > ffHt {
vm.Height = ffHt
return false
} else if rHt < ffHt {
return false
}
// Here is where we actually swap out the Leader with the Audit server
// being promoted
for listIdx, fedServ := range pl.FedServers {
if fedServ.GetChainID().IsSameAs(fullFault.ServerID) {
fmt.Println("FULL FAULT X:", s.FactomNodeName, fullFault.ServerID.String()[:10], fullFault.AuditServerID.String()[:10], s.GetTimestamp().GetTimeSeconds())
pl.FedServers[listIdx] = theAuditReplacement
pl.FedServers[listIdx].SetOnline(true)
pl.AddAuditServer(fedServ.GetChainID())
s.RemoveAuditServer(fullFault.DBHeight, theAuditReplacement.GetChainID())
// After executing the FullFault successfully, we want to reset
// to the default state (No One At Fault)
s.Leader, s.LeaderVMIndex = s.LeaderPL.GetVirtualServers(s.CurrentMinute, s.IdentityChainID)
authoritiesString := s.ConstructAuthoritySetString()
// Any updates required to the state as established by the AdminBlock are applied here.
pl.State.SetAuthoritySetString(authoritiesString)
authorityDeltaString := fmt.Sprintf("Full Fault (DBHt: %d SysHt: %d) \n ^ %s \n v %s", fullFault.DBHeight, fullFault.SystemHeight, fullFault.AuditServerID.String()[5:10], fullFault.ServerID.String()[5:10])
pl.State.AddAuthorityDelta(authorityDeltaString)
pl.Unfault()
haveReplaced = true
break
}
}
}
}
return haveReplaced
}