// startCtfeMetrics registers gauges with the graphite server that indicate CT is running healthily // and starts a goroutine to update them periodically. func startCtfeMetrics() { pendingTasksGauge := metrics.GetOrRegisterGauge("num-pending-tasks", metrics.DefaultRegistry) oldestPendingTaskAgeGauge := metrics.GetOrRegisterGaugeFloat64("oldest-pending-task-age", metrics.DefaultRegistry) // 0=no tasks pending; 1=started; 2=not started oldestPendingTaskStatusGauge := metrics.GetOrRegisterGauge("oldest-pending-task-status", metrics.DefaultRegistry) go func() { for _ = range time.Tick(common.SAMPLE_PERIOD) { pendingTaskCount, err := pending_tasks.GetPendingTaskCount() if err != nil { glog.Error(err) } else { pendingTasksGauge.Update(pendingTaskCount) } oldestPendingTask, err := pending_tasks.GetOldestPendingTask() if err != nil { glog.Error(err) } else if oldestPendingTask == nil { oldestPendingTaskAgeGauge.Update(0) oldestPendingTaskStatusGauge.Update(0) } else { addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(oldestPendingTask.GetCommonCols().TsAdded.Int64, 10)) oldestPendingTaskAgeGauge.Update(time.Since(addedTime).Seconds()) if oldestPendingTask.GetCommonCols().TsStarted.Valid { oldestPendingTaskStatusGauge.Update(1) } else { oldestPendingTaskStatusGauge.Update(2) } } } }() }
// repeatedTasksScheduler looks for all tasks that contain repeat_after_days // set to > 0 and schedules them when the specified time comes. // The function does the following: // 1. Look for tasks that need to be scheduled in the next 5 minutes. // 2. Loop over these tasks. // 2.1 Schedule the task again and set repeat_after_days to what it // originally was. // 2.2 Update the original task and set repeat_after_days to 0 since the // newly created task will now replace it. func repeatedTasksScheduler() { for _ = range time.Tick(*tasksSchedulerWaitTime) { // Loop over all tasks to find tasks which need to be scheduled. for _, prototype := range task_types.Prototypes() { query, args := task_common.DBTaskQuery(prototype, task_common.QueryParams{ FutureRunsOnly: true, Offset: 0, Size: task_common.MAX_PAGE_SIZE, }) glog.Infof("Running %s", query) data, err := prototype.Select(query, args...) if err != nil { glog.Errorf("Failed to query %s tasks: %v", prototype.GetTaskName(), err) continue } tasks := task_common.AsTaskSlice(data) for _, task := range tasks { addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(task.GetCommonCols().TsAdded.Int64, 10)) scheduledTime := addedTime.Add(time.Duration(task.GetCommonCols().RepeatAfterDays) * time.Hour * 24) cutOffTime := time.Now().UTC().Add(*tasksSchedulerWaitTime) if scheduledTime.Before(cutOffTime) { addTaskVars := task.GetPopulatedAddTaskVars() if _, err := task_common.AddTask(addTaskVars); err != nil { glog.Errorf("Failed to add task %v: %v", task, err) continue } taskVars := task.GetUpdateTaskVars() taskVars.GetUpdateTaskCommonVars().Id = task.GetCommonCols().Id taskVars.GetUpdateTaskCommonVars().ClearRepeatAfterDays() if err := task_common.UpdateTask(taskVars, task.TableName()); err != nil { glog.Errorf("Failed to update task %v: %v", task, err) continue } } } } } }