func NewConsulHealthChecker(client consulutil.ConsulClient) ConsulHealthChecker {
return consulHealthChecker{
client: client,
kv: client.KV(),
consulStore: kp.NewConsulStore(client),
}
}
// NOTE: The "retries" concept is mimicking what is built in rcstore.
// TODO: explore transactionality of operations and returning errors instead of
// using retries
func NewConsul(client consulutil.ConsulClient, retries int, logger *logging.Logger) Store {
return &consulStore{
applicator: labels.NewConsulApplicator(client, retries),
kv: client.KV(),
logger: *logger,
}
}
func (rm *P2RM) configureStorage(client consulutil.ConsulClient, labeler labels.ApplicatorWithoutWatches) {
rm.Client = client
rm.Store = kp.NewConsulStore(client)
rm.RCStore = rcstore.NewConsul(client, labeler, 5)
rm.Labeler = labeler
rm.PodStore = podstore.NewConsul(client.KV())
}
func NewConsul(client consulutil.ConsulClient, retries int) *consulStore {
return &consulStore{
retries: retries,
applicator: labels.NewConsulApplicator(client, retries),
kv: client.KV(),
}
}
func NewConsul(client consulutil.ConsulClient, retries int, logger *logging.Logger) Store {
return &consulStore{
retries: retries,
kv: client.KV(),
logger: *logger,
}
}
func NewConsul(client consulutil.ConsulClient, labeler rcLabeler, retries int) *consulStore {
return &consulStore{
retries: retries,
labeler: labeler,
kv: client.KV(),
}
}
func NewConsul(client consulutil.ConsulClient, labeler pcLabeler, watcher pcWatcher, logger *logging.Logger) Store {
return &consulStore{
kv: client.KV(),
logger: *logger,
labeler: labeler,
watcher: watcher,
}
}
func NewConsulApplicator(client consulutil.ConsulClient, retries int) *consulApplicator {
return &consulApplicator{
logger: logging.DefaultLogger,
kv: client.KV(),
retries: retries,
aggregators: map[Type]*consulAggregator{},
retryMetric: metrics.NewGauge(),
}
}
func NewConsulStore(client consulutil.ConsulClient) *consulStore {
statusStore := statusstore.NewConsul(client)
podStatusStore := podstatus.NewConsul(statusStore, PreparerPodStatusNamespace)
podStore := podstore.NewConsul(client.KV())
return &consulStore{
client: client,
podStore: podStore,
podStatusStore: podStatusStore,
}
}
func NewConsul(c consulutil.ConsulClient, labeler rollLabeler, logger *logging.Logger) Store {
if logger == nil {
logger = &logging.DefaultLogger
}
return consulStore{
kv: c.KV(),
rcstore: rcstore.NewConsul(c, labeler, 3),
logger: *logger,
labeler: labeler,
store: kp.NewConsulStore(c),
}
}
func NewConsulStore(client consulutil.ConsulClient) Store {
return &consulStore{
client: client,
podStore: podstore.NewConsul(client.KV()),
}
}
// processHealthUpdater() runs in a goroutine to keep Consul in sync with the local health
// state. It is written as a non-blocking finite state machine: events arrive and update
// internal state, and after each event, the internal state is examined to see if an
// asynchronous action needs to be taken.
//
// Events come from three different sources:
// 1. App monitors send their periodic health check results here. When the service is no
// longer being checked, the monitor must close this channel.
// 2. The session manager sends notifications whenever the current Consul session
// expires or is renewed. When the manager exits, it must close this channel.
// 3. Writes to Consul are performed in a separate goroutine, and when each finishes, it
// notifies the updater of what it just wrote.
//
// In response to these events, two actions can be taken:
// A. Exit, once the app monitor has exited and the health check in Consul has been
// removed.
// B. Write the recent service state to Consul. At most one outstanding write will be
// in-flight at any time.
func processHealthUpdater(
client consulutil.ConsulClient,
checksStream <-chan WatchResult,
sessionsStream <-chan string,
logger logging.Logger,
) {
var localHealth *WatchResult // Health last reported by checker
var remoteHealth *WatchResult // Health last written to Consul
var session string // Current session
var write <-chan writeResult // Future result of an in-flight write
var throttle <-chan time.Time // If set, writes are throttled
// Track and limit all writes to avoid crushing Consul
bucketRefreshRate := time.Minute / time.Duration(*HealthWritesPerMinute)
rateLimiter, err := limit.NewTokenBucket(
*HealthMaxBucketSize,
*HealthMaxBucketSize,
bucketRefreshRate,
)
if err != nil {
panic("invalid token bucket parameters")
}
logger.NoFields().Debug("starting update loop")
for {
// Receive event notification; update internal FSM state
select {
case h, ok := <-checksStream:
// The local health checker sent a new result
if ok {
logger.NoFields().Debug("new health status: ", h.Status)
if !healthEquiv(localHealth, &h) {
msg := fmt.Sprintf("Service %s is now %s", h.Service, h.Status)
if health.Passing.Is(h.Status) {
logger.NoFields().Infoln(msg)
} else {
logger.NoFields().Warnln(msg)
}
}
localHealth = &h
} else {
logger.NoFields().Debug("check stream closed")
checksStream = nil
localHealth = nil
}
case s, ok := <-sessionsStream:
// The active Consul session changed
if ok {
logger.NoFields().Debug("new session: ", s)
} else {
logger.NoFields().Debug("session stream closed")
sessionsStream = nil
}
session = s
// The old health result is deleted when its session expires
remoteHealth = nil
case result := <-write:
// The in-flight write completed
logger.NoFields().Debug("write completed: ", result.OK)
write = nil
if result.OK {
remoteHealth = result.Health
if result.Throttle && throttle == nil {
throttle = time.After(time.Duration(*HealthResumeLimit) * bucketRefreshRate)
logger.NoFields().Warningf("Service %s health is flapping; throttling updates", result.Health.Service)
}
}
case <-throttle:
throttle = nil
logger.NoFields().Warning("health is stable; resuming updates")
}
// Exit
if checksStream == nil && remoteHealth == nil && write == nil {
logger.NoFields().Debug("exiting update loop")
return
}
// Send update to Consul
if !healthEquiv(localHealth, remoteHealth) && session != "" && write == nil &&
throttle == nil {
writeLogger := logger.SubLogger(logrus.Fields{
"session": session,
})
w := make(chan writeResult, 1)
if localHealth == nil {
// Don't wait on the rate limiter when removing the health status
rateLimiter.TryUse(1)
logger.NoFields().Debug("deleting remote health")
key := HealthPath(remoteHealth.Service, remoteHealth.Node)
go sendHealthUpdate(writeLogger, w, nil, false, func() error {
_, err := client.KV().Delete(key, nil)
if err != nil {
return consulutil.NewKVError("delete", key, err)
}
return nil
})
} else {
writeHealth := localHealth
doThrottle := false
if count, _ := rateLimiter.TryUse(1); count <= 1 {
// This is the last update before the throttle will be engaged. Write a special
// message.
logger.NoFields().Debug("writing throttled health")
writeHealth = toThrottled(localHealth)
doThrottle = true
} else {
logger.NoFields().Debug("writing remote health")
}
kv, err := healthToKV(*writeHealth, session)
if err != nil {
// Practically, this should never happen.
logger.WithErrorAndFields(err, logrus.Fields{
"health": *writeHealth,
}).Error("could not serialize health update")
localHealth = nil
continue
}
if remoteHealth == nil {
go sendHealthUpdate(writeLogger, w, localHealth, doThrottle, func() error {
ok, _, err := client.KV().Acquire(kv, nil)
if err != nil {
return consulutil.NewKVError("acquire", kv.Key, err)
}
if !ok {
return fmt.Errorf("write denied")
}
return nil
})
} else {
go sendHealthUpdate(writeLogger, w, localHealth, doThrottle, func() error {
_, err := client.KV().Put(kv, nil)
if err != nil {
return consulutil.NewKVError("put", kv.Key, err)
}
return nil
})
}
}
write = w
}
}
}
func NewConsul(client consulutil.ConsulClient) Store {
return &consulStore{
kv: client.KV(),
}
}
