func NewConsulApplicator(client *api.Client, retries int) *consulApplicator { return &consulApplicator{ logger: logging.DefaultLogger, kv: client.KV(), retries: retries, } }
func NewConsul(client *api.Client, retries int) *consulStore { return &consulStore{ retries: retries, applicator: labels.NewConsulApplicator(client, retries), kv: client.KV(), } }
// ConsulSessionManager continually creates and maintains Consul sessions. It is intended // to be run in its own goroutine. If one session expires, a new one will be created. As // sessions come and go, the session ID (or "" for an expired session) will be sent on the // output channel. // // Parameters: // config: Configuration passed to Consul when creating a new session. // client: The Consul client to use. // output: The channel used for exposing Consul session IDs. This method takes // ownership of this channel and will close it once no new IDs will be created. // done: Close this channel to close the current session (if any) and stop creating // new sessions. // logger: Errors will be logged to this logger. func ConsulSessionManager( config api.SessionEntry, client *api.Client, output chan<- string, done chan struct{}, logger logging.Logger, ) { logger.NoFields().Info("session manager: starting up") for { // Check for exit signal select { case <-done: logger.NoFields().Info("session manager: shutting down") close(output) return default: } // Establish a new session id, _, err := client.Session().CreateNoChecks(&config, nil) if err != nil { logger.WithError(err).Error("session manager: error creating Consul session") time.Sleep(SessionRetryTimeout) continue } sessionLogger := logger.SubLogger(logrus.Fields{ "session": id, }) sessionLogger.NoFields().Info("session manager: new Consul session") select { case output <- id: // Maintain the session err = client.Session().RenewPeriodic(config.TTL, id, nil, done) if err != nil { sessionLogger.WithError(err).Error("session manager: lost session") } else { sessionLogger.NoFields().Info("session manager: released session") } output <- "" case <-done: // Don't bother reporting the new session if exiting client.Session().Destroy(id, nil) sessionLogger.NoFields().Info("session manager: released session") } } }
// processHealthUpdater() runs in a goroutine to keep Consul in sync with the local health // state. It is written as a non-blocking finite state machine: events arrive and update // internal state, and after each event, the internal state is examined to see if an // asynchronous action needs to be taken. // // Events come from three different sources: // 1. App monitors send their periodic health check results here. When the service is no // longer being checked, the monitor must close this channel. // 2. The session manager sends notifications whenever the current Consul session // expires or is renewed. When the manager exits, it must close this channel. // 3. Writes to Consul are performed in a separate goroutine, and when each finishes, it // notifies the updater of what it just wrote. // // In response to these events, two actions can be taken: // A. Exit, once the app monitor has exited and the health check in Consul has been // removed. // B. Write the recent service state to Consul. At most one outstanding write will be // in-flight at any time. func processHealthUpdater( client *api.Client, checksStream <-chan WatchResult, sessionsStream <-chan string, logger logging.Logger, ) { var localHealth *WatchResult // Health last reported by checker var remoteHealth *WatchResult // Health last written to Consul var session string // Current session var write <-chan writeResult // Future result of an in-flight write var throttle <-chan time.Time // If set, writes are throttled // Track and limit all writes to avoid crushing Consul bucketRefreshRate := time.Minute / time.Duration(*HealthWritesPerMinute) rateLimiter, err := limit.NewTokenBucket( *HealthMaxBucketSize, *HealthMaxBucketSize, bucketRefreshRate, ) if err != nil { panic("invalid token bucket parameters") } logger.NoFields().Debug("starting update loop") for { // Receive event notification; update internal FSM state select { case h, ok := <-checksStream: // The local health checker sent a new result if ok { logger.NoFields().Debug("new health status: ", h.Status) localHealth = &h } else { logger.NoFields().Debug("check stream closed") checksStream = nil localHealth = nil } case s, ok := <-sessionsStream: // The active Consul session changed if ok { logger.NoFields().Debug("new session: ", s) } else { logger.NoFields().Debug("session stream closed") sessionsStream = nil } session = s // The old health result is deleted when its session expires remoteHealth = nil case result := <-write: // The in-flight write completed logger.NoFields().Debug("write completed: ", result.OK) write = nil if result.OK { remoteHealth = result.Health if result.Throttle && throttle == nil { throttle = time.After(time.Duration(*HealthResumeLimit) * bucketRefreshRate) logger.NoFields().Warning("health is flapping; throttling updates") } } case <-throttle: throttle = nil logger.NoFields().Warning("health is stable; resuming updates") } // Exit if checksStream == nil && remoteHealth == nil && write == nil { logger.NoFields().Debug("exiting update loop") return } // Send update to Consul if !healthEquiv(localHealth, remoteHealth) && session != "" && write == nil && throttle == nil { writeLogger := logger.SubLogger(logrus.Fields{ "session": session, }) w := make(chan writeResult, 1) if localHealth == nil { // Don't wait on the rate limiter when removing the health status rateLimiter.TryUse(1) logger.NoFields().Debug("deleting remote health") key := HealthPath(remoteHealth.Service, remoteHealth.Node) go sendHealthUpdate(writeLogger, w, nil, false, func() error { _, err := client.KV().Delete(key, nil) if err != nil { return consulutil.NewKVError("delete", key, err) } return nil }) } else { writeHealth := localHealth doThrottle := false if count, _ := rateLimiter.TryUse(1); count <= 1 { // This is the last update before the throttle will be engaged. Write a special // message. logger.NoFields().Debug("writing throttled health") writeHealth = toThrottled(localHealth) doThrottle = true } else { logger.NoFields().Debug("writing remote health") } kv, err := healthToKV(*writeHealth, session) if err != nil { // Practically, this should never happen. logger.WithErrorAndFields(err, logrus.Fields{ "health": *writeHealth, }).Error("could not serialize health update") localHealth = nil continue } if remoteHealth == nil { go sendHealthUpdate(writeLogger, w, localHealth, doThrottle, func() error { ok, _, err := client.KV().Acquire(kv, nil) if err != nil { return consulutil.NewKVError("acquire", kv.Key, err) } if !ok { return fmt.Errorf("write denied") } return nil }) } else { go sendHealthUpdate(writeLogger, w, localHealth, doThrottle, func() error { _, err := client.KV().Put(kv, nil) if err != nil { return consulutil.NewKVError("put", kv.Key, err) } return nil }) } } write = w } } }
func NewConsul(c *api.Client) Store { return consulStore{c.KV()} }