// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
if config.Config.DatabaselessMode__experimental {
log.Fatal("Cannot execute continuous mode in databaseless mode")
}
log.Infof("Starting continuous discovery")
recentDiscoveryOperationKeys = cache.New(time.Duration(config.Config.InstancePollSeconds)*time.Second, time.Second)
inst.LoadHostnameResolveCache()
go handleDiscoveryRequests()
discoveryTick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
instancePollTick := time.Tick(time.Duration(config.Config.InstancePollSeconds) * time.Second)
caretakingTick := time.Tick(time.Minute)
recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second)
var snapshotTopologiesTick <-chan time.Time
if config.Config.SnapshotTopologiesIntervalHours > 0 {
snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour)
}
go ometrics.InitGraphiteMetrics()
go acceptSignals()
if *config.RuntimeCLIFlags.GrabElection {
process.GrabElection()
}
for {
select {
case <-discoveryTick:
go func() {
wasAlreadyElected := atomic.LoadInt64(&isElectedNode)
myIsElectedNode, err := process.AttemptElection()
if err != nil {
log.Errore(err)
}
if myIsElectedNode {
atomic.StoreInt64(&isElectedNode, 1)
} else {
atomic.StoreInt64(&isElectedNode, 0)
}
if myIsElectedNode {
instanceKeys, err := inst.ReadOutdatedInstanceKeys()
if err != nil {
log.Errore(err)
}
log.Debugf("outdated keys: %+v", instanceKeys)
for _, instanceKey := range instanceKeys {
instanceKey := instanceKey
if instanceKey.IsValid() {
discoveryQueue.Push(instanceKey)
}
}
if wasAlreadyElected == 0 {
// Just turned to be leader!
go process.RegisterNode("", "", false)
}
} else {
log.Debugf("Not elected as active node; polling")
}
}()
case <-instancePollTick:
go func() {
// This tick does NOT do instance poll (these are handled by the oversmapling discoveryTick)
// But rather should invoke such routinely operations that need to be as (or roughly as) frequent
// as instance poll
if atomic.LoadInt64(&isElectedNode) == 1 {
go inst.UpdateInstanceRecentRelaylogHistory()
go inst.RecordInstanceCoordinatesHistory()
}
}()
case <-caretakingTick:
// Various periodic internal maintenance tasks
go func() {
if atomic.LoadInt64(&isElectedNode) == 1 {
go inst.RecordInstanceBinlogFileHistory()
go inst.ForgetLongUnseenInstances()
go inst.ForgetUnseenInstancesDifferentlyResolved()
go inst.ForgetExpiredHostnameResolves()
go inst.DeleteInvalidHostnameResolves()
go inst.ReviewUnseenInstances()
go inst.InjectUnseenMasters()
go inst.ResolveUnknownMasterHostnameResolves()
go inst.UpdateClusterAliases()
go inst.ExpireMaintenance()
go inst.ExpireDowntime()
go inst.ExpireCandidateInstances()
go inst.ExpireHostnameUnresolve()
go inst.ExpireClusterDomainName()
go inst.ExpireAudit()
go inst.ExpireMasterPositionEquivalence()
go inst.ExpirePoolInstances()
go inst.FlushNontrivialResolveCacheToDatabase()
go process.ExpireNodesHistory()
go process.ExpireAccessTokens()
} else {
// Take this opportunity to refresh yourself
go inst.LoadHostnameResolveCache()
}
}()
case <-recoveryTick:
go func() {
if atomic.LoadInt64(&isElectedNode) == 1 {
go ClearActiveFailureDetections()
go ClearActiveRecoveries()
go ExpireBlockedRecoveries()
go AcknowledgeCrashedRecoveries()
go inst.ExpireInstanceAnalysisChangelog()
go CheckAndRecover(nil, nil, false)
}
}()
case <-snapshotTopologiesTick:
go func() {
go inst.SnapshotTopologies()
}()
}
}
}
// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
if config.Config.DatabaselessMode__experimental {
log.Fatal("Cannot execute continuous mode in databaseless mode")
}
log.Infof("Starting continuous discovery")
inst.LoadHostnameResolveCache()
go handleDiscoveryRequests()
discoveryTick := time.Tick(time.Duration(config.Config.DiscoveryPollSeconds) * time.Second)
caretakingTick := time.Tick(time.Minute)
recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second)
var snapshotTopologiesTick <-chan time.Time
if config.Config.SnapshotTopologiesIntervalHours > 0 {
snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour)
}
go ometrics.InitGraphiteMetrics()
go acceptSignals()
if *config.RuntimeCLIFlags.GrabElection {
process.GrabElection()
}
for {
select {
case <-discoveryTick:
go func() {
wasAlreadyElected := isElectedNode
if isElectedNode, _ = process.AttemptElection(); isElectedNode {
instanceKeys, _ := inst.ReadOutdatedInstanceKeys()
log.Debugf("outdated keys: %+v", instanceKeys)
for _, instanceKey := range instanceKeys {
instanceKey := instanceKey
go func() { discoveryInstanceKeys <- instanceKey }()
}
if !wasAlreadyElected {
// Just turned to be leader!
go process.RegisterNode("", "", false)
}
} else {
log.Debugf("Not elected as active node; polling")
}
}()
case <-caretakingTick:
// Various periodic internal maintenance tasks
go func() {
if isElectedNode {
go inst.ForgetLongUnseenInstances()
go inst.ForgetUnseenInstancesDifferentlyResolved()
go inst.ForgetExpiredHostnameResolves()
go inst.DeleteInvalidHostnameResolves()
go inst.ReviewUnseenInstances()
go inst.InjectUnseenMasters()
go inst.ResolveUnknownMasterHostnameResolves()
go inst.UpdateClusterAliases()
go inst.ExpireMaintenance()
go inst.ExpireDowntime()
go inst.ExpireCandidateInstances()
go inst.ExpireHostnameUnresolve()
go inst.ExpireClusterDomainName()
go inst.ExpireAudit()
go inst.ExpireMasterPositionEquivalence()
go inst.FlushNontrivialResolveCacheToDatabase()
go process.ExpireNodesHistory()
}
if !isElectedNode {
// Take this opportunity to refresh yourself
go inst.LoadHostnameResolveCache()
}
go inst.ReadClusterAliases()
}()
case <-recoveryTick:
go func() {
if isElectedNode {
go ClearActiveFailureDetections()
go ClearActiveRecoveries()
go ExpireBlockedRecoveries()
go AcknowledgeCrashedRecoveries()
go inst.ExpireInstanceAnalysisChangelog()
go CheckAndRecover(nil, nil, false)
}
}()
case <-snapshotTopologiesTick:
go func() {
go inst.SnapshotTopologies()
}()
}
}
}