// Run triggers periodic syncing of services and checks with Consul. This is
// a long lived go-routine which is stopped during shutdown.
func (c *Syncer) Run() {
sync := time.NewTimer(0)
for {
select {
case <-sync.C:
d := syncInterval - lib.RandomStagger(syncInterval/syncJitter)
sync.Reset(d)
if err := c.SyncServices(); err != nil {
if c.consulAvailable {
c.logger.Printf("[DEBUG] consul.syncer: error in syncing: %v", err)
}
c.consulAvailable = false
} else {
if !c.consulAvailable {
c.logger.Printf("[DEBUG] consul.syncer: syncs succesful")
}
c.consulAvailable = true
}
case <-c.notifySyncCh:
sync.Reset(syncInterval)
case <-c.shutdownCh:
c.Shutdown()
case <-c.notifyShutdownCh:
sync.Stop()
c.logger.Printf("[INFO] consul.syncer: shutting down syncer ")
return
}
}
}
// StartDiskWatcher periodically checks the disk space consumed by the shared
// allocation directory.
func (d *AllocDir) StartDiskWatcher() {
start := time.Now()
sync := time.NewTimer(d.MaxCheckDiskInterval)
defer sync.Stop()
d.running = true
d.watchCh = make(chan struct{})
for {
select {
case <-d.watchCh:
return
case <-sync.C:
if err := d.syncDiskUsage(); err != nil {
log.Printf("[WARN] client: failed to sync disk usage: %v", err)
}
// Calculate the disk ratio.
diskRatio := float64(d.size) / float64(d.MaxSize*structs.BytesInMegabyte)
// Exponentially decrease the interval when the disk ratio increases.
nextInterval := time.Duration(int64(1.0/(0.1*math.Pow(diskRatio, 2))+5)) * time.Second
// Use the maximum interval for the first five minutes or if the
// disk ratio is sufficiently high. Also use the minimum check interval
// if the disk ratio becomes low enough.
if nextInterval < d.MaxCheckDiskInterval || time.Since(start) < d.CheckDiskMaxEnforcePeriod {
nextInterval = d.MaxCheckDiskInterval
} else if nextInterval > d.MinCheckDiskInterval {
nextInterval = d.MinCheckDiskInterval
}
sync.Reset(nextInterval)
}
}
}
// syncConsul removes services of tasks which are no longer in running state
func (c *Client) syncConsul() {
sync := time.NewTicker(consulSyncInterval)
for {
select {
case <-sync.C:
// Give up pruning services if we can't fingerprint Consul
c.configLock.RLock()
_, ok := c.configCopy.Node.Attributes["consul.server"]
c.configLock.RUnlock()
if !ok {
continue
}
services := make(map[string]struct{})
// Get the existing allocs
c.allocLock.RLock()
allocs := make([]*AllocRunner, 0, len(c.allocs))
for _, ar := range c.allocs {
allocs = append(allocs, ar)
}
c.allocLock.RUnlock()
for _, ar := range allocs {
ar.taskStatusLock.RLock()
taskStates := copyTaskStates(ar.taskStates)
ar.taskStatusLock.RUnlock()
for taskName, taskState := range taskStates {
if taskState.State == structs.TaskStateRunning {
if tr, ok := ar.tasks[taskName]; ok {
for _, service := range tr.task.Services {
svcIdentifier := fmt.Sprintf("%s-%s", ar.alloc.ID, tr.task.Name)
services[service.ID(svcIdentifier)] = struct{}{}
}
}
}
}
}
if err := c.consulService.KeepServices(services); err != nil {
c.logger.Printf("[DEBUG] client: error removing services from non-running tasks: %v", err)
}
case <-c.shutdownCh:
sync.Stop()
c.logger.Printf("[INFO] client: shutting down consul sync")
return
}
}
}
// PeriodicSync triggers periodic syncing of services and checks with Consul.
// This is a long lived go-routine which is stopped during shutdown
func (c *ConsulService) PeriodicSync() {
sync := time.NewTicker(syncInterval)
for {
select {
case <-sync.C:
if err := c.performSync(); err != nil {
if c.availble {
c.logger.Printf("[DEBUG] consul: error in syncing task %q: %v", c.task.Name, err)
}
c.availble = false
} else {
c.availble = true
}
case <-c.shutdownCh:
sync.Stop()
c.logger.Printf("[INFO] consul: shutting down sync for task %q", c.task.Name)
return
}
}
}
