// startCtfeMetrics registers gauges with the graphite server that indicate CT is running healthily
// and starts a goroutine to update them periodically.
func startCtfeMetrics() {
pendingTasksGauge := metrics.GetOrRegisterGauge("num-pending-tasks", metrics.DefaultRegistry)
oldestPendingTaskAgeGauge := metrics.GetOrRegisterGaugeFloat64("oldest-pending-task-age", metrics.DefaultRegistry)
// 0=no tasks pending; 1=started; 2=not started
oldestPendingTaskStatusGauge := metrics.GetOrRegisterGauge("oldest-pending-task-status", metrics.DefaultRegistry)
go func() {
for _ = range time.Tick(common.SAMPLE_PERIOD) {
pendingTaskCount, err := pending_tasks.GetPendingTaskCount()
if err != nil {
glog.Error(err)
} else {
pendingTasksGauge.Update(pendingTaskCount)
}
oldestPendingTask, err := pending_tasks.GetOldestPendingTask()
if err != nil {
glog.Error(err)
} else if oldestPendingTask == nil {
oldestPendingTaskAgeGauge.Update(0)
oldestPendingTaskStatusGauge.Update(0)
} else {
addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(oldestPendingTask.GetCommonCols().TsAdded.Int64, 10))
oldestPendingTaskAgeGauge.Update(time.Since(addedTime).Seconds())
if oldestPendingTask.GetCommonCols().TsStarted.Valid {
oldestPendingTaskStatusGauge.Update(1)
} else {
oldestPendingTaskStatusGauge.Update(2)
}
}
}
}()
}
// repeatedTasksScheduler looks for all tasks that contain repeat_after_days
// set to > 0 and schedules them when the specified time comes.
// The function does the following:
// 1. Look for tasks that need to be scheduled in the next 5 minutes.
// 2. Loop over these tasks.
// 2.1 Schedule the task again and set repeat_after_days to what it
// originally was.
// 2.2 Update the original task and set repeat_after_days to 0 since the
// newly created task will now replace it.
func repeatedTasksScheduler() {
for _ = range time.Tick(*tasksSchedulerWaitTime) {
// Loop over all tasks to find tasks which need to be scheduled.
for _, prototype := range task_types.Prototypes() {
query, args := task_common.DBTaskQuery(prototype,
task_common.QueryParams{
FutureRunsOnly: true,
Offset: 0,
Size: task_common.MAX_PAGE_SIZE,
})
glog.Infof("Running %s", query)
data, err := prototype.Select(query, args...)
if err != nil {
glog.Errorf("Failed to query %s tasks: %v", prototype.GetTaskName(), err)
continue
}
tasks := task_common.AsTaskSlice(data)
for _, task := range tasks {
addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(task.GetCommonCols().TsAdded.Int64, 10))
scheduledTime := addedTime.Add(time.Duration(task.GetCommonCols().RepeatAfterDays) * time.Hour * 24)
cutOffTime := time.Now().UTC().Add(*tasksSchedulerWaitTime)
if scheduledTime.Before(cutOffTime) {
addTaskVars := task.GetPopulatedAddTaskVars()
if _, err := task_common.AddTask(addTaskVars); err != nil {
glog.Errorf("Failed to add task %v: %v", task, err)
continue
}
taskVars := task.GetUpdateTaskVars()
taskVars.GetUpdateTaskCommonVars().Id = task.GetCommonCols().Id
taskVars.GetUpdateTaskCommonVars().ClearRepeatAfterDays()
if err := task_common.UpdateTask(taskVars, task.TableName()); err != nil {
glog.Errorf("Failed to update task %v: %v", task, err)
continue
}
}
}
}
}
}
