func NewConsulApplicator(client *api.Client, retries int) *consulApplicator {
return &consulApplicator{
logger: logging.DefaultLogger,
kv: client.KV(),
retries: retries,
}
}
func NewConsul(client *api.Client, retries int) *consulStore {
return &consulStore{
retries: retries,
applicator: labels.NewConsulApplicator(client, retries),
kv: client.KV(),
}
}
// processHealthUpdater() runs in a goroutine to keep Consul in sync with the local health
// state. It is written as a non-blocking finite state machine: events arrive and update
// internal state, and after each event, the internal state is examined to see if an
// asynchronous action needs to be taken.
//
// Events come from three different sources:
// 1. App monitors send their periodic health check results here. When the service is no
// longer being checked, the monitor must close this channel.
// 2. The session manager sends notifications whenever the current Consul session
// expires or is renewed. When the manager exits, it must close this channel.
// 3. Writes to Consul are performed in a separate goroutine, and when each finishes, it
// notifies the updater of what it just wrote.
//
// In response to these events, two actions can be taken:
// A. Exit, once the app monitor has exited and the health check in Consul has been
// removed.
// B. Write the recent service state to Consul. At most one outstanding write will be
// in-flight at any time.
func processHealthUpdater(
client *api.Client,
checksStream <-chan WatchResult,
sessionsStream <-chan string,
logger logging.Logger,
) {
var localHealth *WatchResult // Health last reported by checker
var remoteHealth *WatchResult // Health last written to Consul
var session string // Current session
var write <-chan writeResult // Future result of an in-flight write
var throttle <-chan time.Time // If set, writes are throttled
// Track and limit all writes to avoid crushing Consul
bucketRefreshRate := time.Minute / time.Duration(*HealthWritesPerMinute)
rateLimiter, err := limit.NewTokenBucket(
*HealthMaxBucketSize,
*HealthMaxBucketSize,
bucketRefreshRate,
)
if err != nil {
panic("invalid token bucket parameters")
}
logger.NoFields().Debug("starting update loop")
for {
// Receive event notification; update internal FSM state
select {
case h, ok := <-checksStream:
// The local health checker sent a new result
if ok {
logger.NoFields().Debug("new health status: ", h.Status)
localHealth = &h
} else {
logger.NoFields().Debug("check stream closed")
checksStream = nil
localHealth = nil
}
case s, ok := <-sessionsStream:
// The active Consul session changed
if ok {
logger.NoFields().Debug("new session: ", s)
} else {
logger.NoFields().Debug("session stream closed")
sessionsStream = nil
}
session = s
// The old health result is deleted when its session expires
remoteHealth = nil
case result := <-write:
// The in-flight write completed
logger.NoFields().Debug("write completed: ", result.OK)
write = nil
if result.OK {
remoteHealth = result.Health
if result.Throttle && throttle == nil {
throttle = time.After(time.Duration(*HealthResumeLimit) * bucketRefreshRate)
logger.NoFields().Warning("health is flapping; throttling updates")
}
}
case <-throttle:
throttle = nil
logger.NoFields().Warning("health is stable; resuming updates")
}
// Exit
if checksStream == nil && remoteHealth == nil && write == nil {
logger.NoFields().Debug("exiting update loop")
return
}
// Send update to Consul
if !healthEquiv(localHealth, remoteHealth) && session != "" && write == nil &&
throttle == nil {
writeLogger := logger.SubLogger(logrus.Fields{
"session": session,
})
w := make(chan writeResult, 1)
if localHealth == nil {
// Don't wait on the rate limiter when removing the health status
rateLimiter.TryUse(1)
logger.NoFields().Debug("deleting remote health")
key := HealthPath(remoteHealth.Service, remoteHealth.Node)
go sendHealthUpdate(writeLogger, w, nil, false, func() error {
_, err := client.KV().Delete(key, nil)
if err != nil {
return consulutil.NewKVError("delete", key, err)
}
return nil
})
} else {
writeHealth := localHealth
doThrottle := false
if count, _ := rateLimiter.TryUse(1); count <= 1 {
// This is the last update before the throttle will be engaged. Write a special
// message.
logger.NoFields().Debug("writing throttled health")
writeHealth = toThrottled(localHealth)
doThrottle = true
} else {
logger.NoFields().Debug("writing remote health")
}
kv, err := healthToKV(*writeHealth, session)
if err != nil {
// Practically, this should never happen.
logger.WithErrorAndFields(err, logrus.Fields{
"health": *writeHealth,
}).Error("could not serialize health update")
localHealth = nil
continue
}
if remoteHealth == nil {
go sendHealthUpdate(writeLogger, w, localHealth, doThrottle, func() error {
ok, _, err := client.KV().Acquire(kv, nil)
if err != nil {
return consulutil.NewKVError("acquire", kv.Key, err)
}
if !ok {
return fmt.Errorf("write denied")
}
return nil
})
} else {
go sendHealthUpdate(writeLogger, w, localHealth, doThrottle, func() error {
_, err := client.KV().Put(kv, nil)
if err != nil {
return consulutil.NewKVError("put", kv.Key, err)
}
return nil
})
}
}
write = w
}
}
}
func NewConsul(c *api.Client) Store {
return consulStore{c.KV()}
}