func (t *T) Set(f FlagType) { t.Flags[f] = struct{}{} if Launched == f { t.launchTime = time.Now() queueWaitTime := t.launchTime.Sub(t.CreateTime) metrics.QueueWaitTime.Observe(metrics.InMicroseconds(queueWaitTime)) } }
func (k *inMemoryRegistry) handleTaskStarting(task *T, state StateType, status *mesos.TaskStatus) { // we expect to receive this when a launched task is finally "bound" // via the API server. however, there's nothing specific for us to do here. switch state { case StatePending: task.UpdatedTime = time.Now() if !task.Has(Bound) { task.Set(Bound) task.bindTime = task.UpdatedTime timeToBind := task.bindTime.Sub(task.launchTime) metrics.BindLatency.Observe(metrics.InMicroseconds(timeToBind)) } default: taskId := status.GetTaskId().GetValue() log.Warningf("Ignore status TASK_STARTING because the task %v is not pending", taskId) } }
// execute task reconciliation, returns when r.done is closed. intended to run as a goroutine. // if reconciliation is requested while another is in progress, the in-progress operation will be // cancelled before the new reconciliation operation begins. func (r *Reconciler) Run(driver bindings.SchedulerDriver) { var cancel, finished chan struct{} requestLoop: for { select { case <-r.done: return default: // proceed } select { case <-r.implicit: metrics.ReconciliationRequested.WithLabelValues("implicit").Inc() select { case <-r.done: return case <-r.explicit: break // give preference to a pending request for explicit default: // continue // don't run implicit reconciliation while explicit is ongoing if finished != nil { select { case <-finished: // continue w/ implicit default: log.Infoln("skipping implicit reconcile because explicit reconcile is ongoing") continue requestLoop } } errOnce := proc.NewErrorOnce(r.done) errCh := r.Do(func() { var err error defer errOnce.Report(err) log.Infoln("implicit reconcile tasks") metrics.ReconciliationExecuted.WithLabelValues("implicit").Inc() if _, err = driver.ReconcileTasks([]*mesos.TaskStatus{}); err != nil { log.V(1).Infof("failed to request implicit reconciliation from mesos: %v", err) } }) proc.OnError(errOnce.Send(errCh).Err(), func(err error) { log.Errorf("failed to run implicit reconciliation: %v", err) }, r.done) goto slowdown } case <-r.done: return case <-r.explicit: // continue metrics.ReconciliationRequested.WithLabelValues("explicit").Inc() } if cancel != nil { close(cancel) cancel = nil // play nice and wait for the prior operation to finish, complain // if it doesn't select { case <-r.done: return case <-finished: // noop, expected case <-time.After(r.explicitReconciliationAbortTimeout): // very unexpected log.Error("reconciler action failed to stop upon cancellation") } } // copy 'finished' to 'fin' here in case we end up with simultaneous go-routines, // if cancellation takes too long or fails - we don't want to close the same chan // more than once cancel = make(chan struct{}) finished = make(chan struct{}) go func(fin chan struct{}) { startedAt := time.Now() defer func() { metrics.ReconciliationLatency.Observe(metrics.InMicroseconds(time.Since(startedAt))) }() metrics.ReconciliationExecuted.WithLabelValues("explicit").Inc() defer close(fin) err := <-r.Action(driver, cancel) if err == reconciliationCancelledErr { metrics.ReconciliationCancelled.WithLabelValues("explicit").Inc() log.Infoln(err.Error()) } else if err != nil { log.Errorf("reconciler action failed: %v", err) } }(finished) slowdown: // don't allow reconciliation to run very frequently, either explicit or implicit select { case <-r.done: return case <-time.After(r.cooldown): // noop } } // for }