// standardHttp starts serving standard HTTP (api/web) requests, to be used by normal clients func standardHttp(discovery bool) { m := martini.Classic() switch strings.ToLower(config.Config.AuthenticationMethod) { case "basic": { if config.Config.HTTPAuthUser == "" { // Still allowed; may be disallowed in future versions log.Warning("AuthenticationMethod is configured as 'basic' but HTTPAuthUser undefined. Running without authentication.") } m.Use(auth.Basic(config.Config.HTTPAuthUser, config.Config.HTTPAuthPassword)) } case "multi": { if config.Config.HTTPAuthUser == "" { // Still allowed; may be disallowed in future versions log.Fatal("AuthenticationMethod is configured as 'multi' but HTTPAuthUser undefined") } m.Use(auth.BasicFunc(func(username, password string) bool { if username == "readonly" { // Will be treated as "read-only" return true } return auth.SecureCompare(username, config.Config.HTTPAuthUser) && auth.SecureCompare(password, config.Config.HTTPAuthPassword) })) } default: { // We inject a dummy User object because we have function signatures with User argument in api.go m.Map(auth.User("")) } } m.Use(gzip.All()) // Render html templates from templates directory m.Use(render.Renderer(render.Options{ Directory: "resources", Layout: "templates/layout", HTMLContentType: "text/html", })) m.Use(martini.Static("resources/public")) inst.SetMaintenanceOwner(logic.ThisHostname) log.Info("Starting HTTP") if discovery { go logic.ContinuousDiscovery() } inst.ReadClusterAliases() http.API.RegisterRequests(m) http.Web.RegisterRequests(m) // Serve if err := nethttp.ListenAndServe(config.Config.ListenAddress, m); err != nil { log.Fatale(err) } }
// DiscoverInstance will attempt discovering an instance (unless it is already up to date) and will // list down its master and slaves (if any) for further discovery. func DiscoverInstance(instanceKey inst.InstanceKey) { instanceKey.Formalize() if !instanceKey.IsValid() { return } instance, found, err := inst.ReadInstance(&instanceKey) if found && instance.IsUpToDate && instance.IsLastCheckValid { // we've already discovered this one. Skip! goto Cleanup } // First we've ever heard of this instance. Continue investigation: instance, err = inst.ReadTopologyInstance(&instanceKey) // panic can occur (IO stuff). Therefore it may happen // that instance is nil. Check it. if err != nil || instance == nil { log.Warningf("instance is nil in DiscoverInstance. key=%+v, error=%+v", instanceKey, err) goto Cleanup } log.Debugf("Discovered host: %+v, master: %+v", instance.Key, instance.MasterKey) // Investigate slaves: for _, slaveKey := range instance.SlaveHosts.GetInstanceKeys() { discoveryInstanceKeys <- slaveKey } // Investigate master: discoveryInstanceKeys <- instance.MasterKey Cleanup: } // Start discovery begins a one time asynchronuous discovery process for the given // instance and all of its topology connected instances. // That is, the instance will be investigated for master and slaves, and the routines will follow on // each and every such found master/slave. // In essense, assuming all slaves in a replication topology are running, and given a single instance // in such topology, this function will detect the entire topology. func StartDiscovery(instanceKey inst.InstanceKey) { log.Infof("Starting discovery at %+v", instanceKey) pendingTokens := make(chan bool, maxConcurrency) completedTokens := make(chan bool, maxConcurrency) AccountedDiscoverInstance(instanceKey, pendingTokens, completedTokens) go handleDiscoveryRequests(pendingTokens, completedTokens) // Block until all are complete for { select { case <-pendingTokens: <-completedTokens default: inst.AuditOperation("start-discovery", &instanceKey, "") return } } } // ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are // periodically investigated and their status captured, and long since unseen instances are // purged and forgotten. func ContinuousDiscovery() { log.Infof("Starting continuous discovery") inst.LoadHostnameResolveCacheFromDatabase() go handleDiscoveryRequests(nil, nil) tick := time.Tick(time.Duration(config.Config.DiscoveryPollSeconds) * time.Second) forgetUnseenTick := time.Tick(time.Minute) recoverTick := time.Tick(10 * time.Second) var snapshotTopologiesTick <-chan time.Time if config.Config.SnapshotTopologiesIntervalHours > 0 { snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour) } elected := false _ = CreateElectionAnchor(false) for { select { case <-tick: if elected, _ = AttemptElection(); elected { instanceKeys, _ := inst.ReadOutdatedInstanceKeys() log.Debugf("outdated keys: %+v", instanceKeys) for _, instanceKey := range instanceKeys { discoveryInstanceKeys <- instanceKey } } else { log.Debugf("Not elected as active node; polling") } case <-forgetUnseenTick: // See if we should also forget objects (lower frequency) go func() { if elected { inst.ForgetLongUnseenInstances() inst.ForgetUnseenInstancesDifferentlyResolved() inst.ForgetExpiredHostnameResolves() inst.DeleteInvalidHostnameResolves() inst.ReviewUnseenInstances() inst.InjectUnseenMasters() inst.ResolveUnknownMasterHostnameResolves() inst.ExpireMaintenance() inst.ExpireDowntime() inst.ExpireCandidateInstances() inst.ExpireHostnameUnresolve() inst.ExpireClusterDomainName() } if !elected { // Take this opportunity to refresh yourself inst.LoadHostnameResolveCacheFromDatabase() } inst.ReadClusterAliases() HealthTest() }() case <-recoverTick: go func() { if elected { ClearActiveFailureDetections() ClearActiveRecoveries() CheckAndRecover(nil, nil, false, false) } }() case <-snapshotTopologiesTick: go func() { inst.SnapshotTopologies() }() } } } func pollAgent(hostname string) error { polledAgent, err := agent.GetAgent(hostname) agent.UpdateAgentLastChecked(hostname) if err != nil { return log.Errore(err) } err = agent.UpdateAgentInfo(hostname, polledAgent) if err != nil { return log.Errore(err) } return nil } // ContinuousAgentsPoll starts an asynchronuous infinite process where agents are // periodically investigated and their status captured, and long since unseen agents are // purged and forgotten. func ContinuousAgentsPoll() { log.Infof("Starting continuous agents poll") go discoverSeededAgents() tick := time.Tick(time.Duration(config.Config.DiscoveryPollSeconds) * time.Second) forgetUnseenTick := time.Tick(time.Hour) for _ = range tick { agentsHosts, _ := agent.ReadOutdatedAgentsHosts() log.Debugf("outdated agents hosts: %+v", agentsHosts) for _, hostname := range agentsHosts { go pollAgent(hostname) } // See if we should also forget agents (lower frequency) select { case <-forgetUnseenTick: agent.ForgetLongUnseenAgents() agent.FailStaleSeeds() default: } } } func discoverSeededAgents() { for seededAgent := range agent.SeededAgents { instanceKey := inst.InstanceKey{Hostname: seededAgent.Hostname, Port: int(seededAgent.MySQLPort)} go StartDiscovery(instanceKey) } }
// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are // periodically investigated and their status captured, and long since unseen instances are // purged and forgotten. func ContinuousDiscovery() { if config.Config.DatabaselessMode__experimental { log.Fatal("Cannot execute continuous mode in databaseless mode") } log.Infof("Starting continuous discovery") inst.LoadHostnameResolveCacheFromDatabase() go handleDiscoveryRequests() discoveryTick := time.Tick(time.Duration(config.Config.DiscoveryPollSeconds) * time.Second) caretakingTick := time.Tick(time.Minute) recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second) var snapshotTopologiesTick <-chan time.Time if config.Config.SnapshotTopologiesIntervalHours > 0 { snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour) } go ometrics.InitGraphiteMetrics() go acceptSignals() for { select { case <-discoveryTick: go func() { wasAlreadyElected := isElectedNode if isElectedNode, _ = process.AttemptElection(); isElectedNode { instanceKeys, _ := inst.ReadOutdatedInstanceKeys() log.Debugf("outdated keys: %+v", instanceKeys) for _, instanceKey := range instanceKeys { go func() { discoveryInstanceKeys <- instanceKey }() } if !wasAlreadyElected { // Just turned to be leader! go process.RegisterNode("", "", false) } } else { log.Debugf("Not elected as active node; polling") } }() case <-caretakingTick: // Various periodic internal maintenance tasks go func() { if isElectedNode { go inst.ForgetLongUnseenInstances() go inst.ForgetUnseenInstancesDifferentlyResolved() go inst.ForgetExpiredHostnameResolves() go inst.DeleteInvalidHostnameResolves() go inst.ReviewUnseenInstances() go inst.InjectUnseenMasters() go inst.ResolveUnknownMasterHostnameResolves() go inst.UpdateClusterAliases() go inst.ExpireMaintenance() go inst.ExpireDowntime() go inst.ExpireCandidateInstances() go inst.ExpireHostnameUnresolve() go inst.ExpireClusterDomainName() go inst.ExpireAudit() go inst.ExpireMasterPositionEquivalence() go inst.FlushNontrivialResolveCacheToDatabase() go process.ExpireNodesHistory() } if !isElectedNode { // Take this opportunity to refresh yourself go inst.LoadHostnameResolveCacheFromDatabase() } go inst.ReadClusterAliases() }() case <-recoveryTick: go func() { if isElectedNode { go ClearActiveFailureDetections() go ClearActiveRecoveries() go ExpireBlockedRecoveries() go AcknowledgeCrashedRecoveries() go inst.ExpireInstanceAnalysisChangelog() go CheckAndRecover(nil, nil, false) } }() case <-snapshotTopologiesTick: go func() { go inst.SnapshotTopologies() }() } } }
// standardHttp starts serving HTTP or HTTPS (api/web) requests, to be used by normal clients func standardHttp(discovery bool) { m := martini.Classic() switch strings.ToLower(config.Config.AuthenticationMethod) { case "basic": { if config.Config.HTTPAuthUser == "" { // Still allowed; may be disallowed in future versions log.Warning("AuthenticationMethod is configured as 'basic' but HTTPAuthUser undefined. Running without authentication.") } m.Use(auth.Basic(config.Config.HTTPAuthUser, config.Config.HTTPAuthPassword)) } case "multi": { if config.Config.HTTPAuthUser == "" { // Still allowed; may be disallowed in future versions log.Fatal("AuthenticationMethod is configured as 'multi' but HTTPAuthUser undefined") } m.Use(auth.BasicFunc(func(username, password string) bool { if username == "readonly" { // Will be treated as "read-only" return true } return auth.SecureCompare(username, config.Config.HTTPAuthUser) && auth.SecureCompare(password, config.Config.HTTPAuthPassword) })) } default: { // We inject a dummy User object because we have function signatures with User argument in api.go m.Map(auth.User("")) } } m.Use(gzip.All()) // Render html templates from templates directory m.Use(render.Renderer(render.Options{ Directory: "resources", Layout: "templates/layout", HTMLContentType: "text/html", })) m.Use(martini.Static("resources/public")) if config.Config.UseMutualTLS { m.Use(ssl.VerifyOUs(config.Config.SSLValidOUs)) } inst.SetMaintenanceOwner(process.ThisHostname) if discovery { log.Info("Starting Discovery") go logic.ContinuousDiscovery() } inst.ReadClusterAliases() log.Info("Registering endpoints") http.API.RegisterRequests(m) http.Web.RegisterRequests(m) // Serve if config.Config.UseSSL { log.Info("Starting HTTPS listener") tlsConfig, err := ssl.NewTLSConfig(config.Config.SSLCAFile, config.Config.UseMutualTLS) if err != nil { log.Fatale(err) } tlsConfig.InsecureSkipVerify = config.Config.SSLSkipVerify if err = ssl.AppendKeyPair(tlsConfig, config.Config.SSLCertFile, config.Config.SSLPrivateKeyFile); err != nil { log.Fatale(err) } if err = ssl.ListenAndServeTLS(config.Config.ListenAddress, m, tlsConfig); err != nil { log.Fatale(err) } } else { log.Info("Starting HTTP listener") if err := nethttp.ListenAndServe(config.Config.ListenAddress, m); err != nil { log.Fatale(err) } } log.Info("Web server started") }
// discoverInstance will attempt discovering an instance (unless it is already up to date) and will // list down its master and slaves (if any) for further discovery. func discoverInstance(instanceKey inst.InstanceKey) { instanceKey.Formalize() if !instanceKey.IsValid() { return } instance, found, err := inst.ReadInstance(&instanceKey) if found && instance.IsUpToDate && instance.IsLastCheckValid { // we've already discovered this one. Skip! goto Cleanup } discoveriesCounter.Inc(1) // First we've ever heard of this instance. Continue investigation: instance, err = inst.ReadTopologyInstance(&instanceKey) // panic can occur (IO stuff). Therefore it may happen // that instance is nil. Check it. if err != nil || instance == nil { failedDiscoveriesCounter.Inc(1) log.Warningf("instance is nil in discoverInstance. key=%+v, error=%+v", instanceKey, err) goto Cleanup } log.Debugf("Discovered host: %+v, master: %+v", instance.Key, instance.MasterKey) if !isElectedNode { // Maybe this node was elected before, but isn't elected anymore. // If not elected, stop drilling down to further investigate slaves. return } // Investigate slaves: for _, slaveKey := range instance.SlaveHosts.GetInstanceKeys() { discoveryInstanceKeys <- slaveKey } // Investigate master: discoveryInstanceKeys <- instance.MasterKey Cleanup: } // Start discovery begins a one time asynchronuous discovery process for the given // instance and all of its topology connected instances. // That is, the instance will be investigated for master and slaves, and the routines will follow on // each and every such found master/slave. // In essense, assuming all slaves in a replication topology are running, and given a single instance // in such topology, this function will detect the entire topology. func StartDiscovery(instanceKey inst.InstanceKey) { log.Infof("Starting discovery at %+v", instanceKey) pendingTokens := make(chan bool, maxConcurrency) completedTokens := make(chan bool, maxConcurrency) accountedDiscoverInstance(instanceKey, pendingTokens, completedTokens) go handleDiscoveryRequests(pendingTokens, completedTokens) // Block until all are complete for { select { case <-pendingTokens: <-completedTokens default: inst.AuditOperation("start-discovery", &instanceKey, "") return } } } func initGraphiteMetrics() error { if config.Config.GraphiteAddr == "" { return nil } if config.Config.GraphitePath == "" { return log.Errorf("No graphite path provided (see GraphitePath config variable). Will not log to graphite") } addr, err := net.ResolveTCPAddr("tcp", config.Config.GraphiteAddr) if err != nil { return log.Errore(err) } graphitePathHostname := ThisHostname if config.Config.GraphiteConvertHostnameDotsToUnderscores { graphitePathHostname = strings.Replace(graphitePathHostname, ".", "_", -1) } graphitePath := config.Config.GraphitePath graphitePath = strings.Replace(graphitePath, "{hostname}", graphitePathHostname, -1) log.Debugf("Will log to graphite on %+v, %+v", config.Config.GraphiteAddr, graphitePath) go graphite.Graphite(metrics.DefaultRegistry, 1*time.Minute, graphitePath, addr) return nil } // ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are // periodically investigated and their status captured, and long since unseen instances are // purged and forgotten. func ContinuousDiscovery() { log.Infof("Starting continuous discovery") inst.LoadHostnameResolveCacheFromDatabase() go handleDiscoveryRequests(nil, nil) tick := time.Tick(time.Duration(config.Config.DiscoveryPollSeconds) * time.Second) forgetUnseenTick := time.Tick(time.Minute) recoverTick := time.Tick(10 * time.Second) var snapshotTopologiesTick <-chan time.Time if config.Config.SnapshotTopologiesIntervalHours > 0 { snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour) } go initGraphiteMetrics() for { select { case <-tick: go func() { if isElectedNode, _ = attemptElection(); isElectedNode { instanceKeys, _ := inst.ReadOutdatedInstanceKeys() log.Debugf("outdated keys: %+v", instanceKeys) for _, instanceKey := range instanceKeys { discoveryInstanceKeys <- instanceKey } } else { log.Debugf("Not elected as active node; polling") } discoveryQueueLengthGauge.Update(int64(len(discoveryInstanceKeys))) }() case <-forgetUnseenTick: // See if we should also forget objects (lower frequency) go func() { if isElectedNode { inst.ForgetLongUnseenInstances() inst.ForgetUnseenInstancesDifferentlyResolved() inst.ForgetExpiredHostnameResolves() inst.DeleteInvalidHostnameResolves() inst.ReviewUnseenInstances() inst.InjectUnseenMasters() inst.ResolveUnknownMasterHostnameResolves() inst.ExpireMaintenance() inst.ExpireDowntime() inst.ExpireCandidateInstances() inst.ExpireHostnameUnresolve() inst.ExpireClusterDomainName() inst.ExpireAudit() inst.ExpireMasterPositionEquivalence() } if !isElectedNode { // Take this opportunity to refresh yourself inst.LoadHostnameResolveCacheFromDatabase() } inst.ReadClusterAliases() HealthTest() }() case <-recoverTick: go func() { if isElectedNode { ClearActiveFailureDetections() ClearActiveRecoveries() CheckAndRecover(nil, nil, false, false) } }() case <-snapshotTopologiesTick: go func() { inst.SnapshotTopologies() }() } } } func pollAgent(hostname string) error { polledAgent, err := agent.GetAgent(hostname) agent.UpdateAgentLastChecked(hostname) if err != nil { return log.Errore(err) } err = agent.UpdateAgentInfo(hostname, polledAgent) if err != nil { return log.Errore(err) } return nil } // ContinuousAgentsPoll starts an asynchronuous infinite process where agents are // periodically investigated and their status captured, and long since unseen agents are // purged and forgotten. func ContinuousAgentsPoll() { log.Infof("Starting continuous agents poll") go discoverSeededAgents() tick := time.Tick(time.Duration(config.Config.DiscoveryPollSeconds) * time.Second) forgetUnseenTick := time.Tick(time.Hour) for range tick { agentsHosts, _ := agent.ReadOutdatedAgentsHosts() log.Debugf("outdated agents hosts: %+v", agentsHosts) for _, hostname := range agentsHosts { go pollAgent(hostname) } // See if we should also forget agents (lower frequency) select { case <-forgetUnseenTick: agent.ForgetLongUnseenAgents() agent.FailStaleSeeds() default: } } } func discoverSeededAgents() { for seededAgent := range agent.SeededAgents { instanceKey := inst.InstanceKey{Hostname: seededAgent.Hostname, Port: int(seededAgent.MySQLPort)} go StartDiscovery(instanceKey) } }