func loadSettingsFromRows(jiRows map[string]map[string]*JobInfoEntry, scripts map[string]*ScriptEntry) error {
newIds := make(map[uint64]bool)
jiRowsCnt := 0
func() {
allSettingsMutex.Lock()
defer allSettingsMutex.Unlock()
for _, row := range scripts {
if _, ok := allSettings[row.settings_id]; !ok {
newIds[row.settings_id] = true
}
}
for _, rows := range jiRows {
for _, row := range rows {
jiRowsCnt++
if _, ok := allSettings[row.settings_id]; !ok {
newIds[row.settings_id] = true
}
}
}
}()
if len(newIds) > 0 {
loadNewIdsTs := time.Now().UnixNano()
err := loadNewIds(newIds)
if err != nil {
return err
}
log.Printf("Loaded %d new ids for %.5f sec", len(newIds), float64(time.Now().UnixNano()-loadNewIdsTs)/1e9)
}
log.Debugln(" Selected", len(scripts), "rows from scripts")
log.Debugln(" Selected", jiRowsCnt, "rows from job info")
return nil
}
func (d *DispatcherData) redispatch() {
returnToWaitingList := make([]*TimetableEntry, 0)
defer func() {
for _, row := range returnToWaitingList {
d.addToWaiting(row)
}
}()
now := uint64(time.Now().Unix())
newRqList := make([]*RunQueueEntry, 0)
toDeleteFromWaitingList := make([]*TimetableEntry, 0)
for l := d.waitingList.Len(); l > 0; l-- {
row := heap.Pop(&d.waitingList).(*TimetableEntry)
delete(d.waitingMap, row.id)
if d.killRequest != nil {
toDeleteFromWaitingList = append(toDeleteFromWaitingList, row)
continue
}
if uint64(row.NextLaunchTs.Int64) > now {
d.tickRedispatchCh = time.After(time.Second * time.Duration(uint64(row.NextLaunchTs.Int64)-now))
returnToWaitingList = append(returnToWaitingList, row)
break
}
if len(d.addedMap) >= row.settings.instance_count {
returnToWaitingList = append(returnToWaitingList, row)
break
}
if _, ok := d.addedJobData[row.JobData]; ok {
if !row.reportedDup {
log.Warningf("Duplicate job %s for class %s and location %s", row.JobData, d.className, row.location)
row.reportedDup = true
}
returnToWaitingList = append(returnToWaitingList, row)
continue
}
if row.method == METHOD_RUN && row.settings.ttl > 0 && now > row.created+uint64(row.settings.ttl) {
if row.finish_count == 0 {
log.Warningf("Job expired before being run even once: job %s for class %s and location %s", row.JobData, d.className, row.location)
}
toDeleteFromWaitingList = append(toDeleteFromWaitingList, row)
continue
}
// do not try to dispatch next ones if selectHostname failed, and do not forget to return the row as well
hostname, err := selectHostname(row.location, row.settings.location_type, d.rusage.cpu_usage, d.rusage.max_memory)
if err != nil {
logFailedLocation(row.settings, row.location, err.Error())
d.tickRedispatchCh = time.After(time.Second)
returnToWaitingList = append(returnToWaitingList, row)
break
} else {
settings := row.settings
if settings.location_type == LOCATION_TYPE_ANY && (settings.developer.String != "") && (settings.developer.String != "wwwrun") && ((now - uint64(settings.created)) <= DEVELOPER_CUSTOM_PATH_TIMEOUT) {
hostname = DEVELOPER_DEBUG_HOSTNAME
}
log.Debugln("Selected ", hostname, " for ", row.location, " (loc_type=", settings.location_type, ")")
}
nullNow := sql.NullInt64{Valid: true, Int64: int64(now)}
queueRow := &RunQueueEntry{
ClassName: d.className,
timetable_id: sql.NullInt64{Valid: true, Int64: int64(row.id)},
generation_id: row.generation_id,
hostname: hostname,
hostname_idx: getHostnameIdx(hostname),
JobData: row.JobData,
method: row.method,
created: nullNow,
RunStatus: RUN_STATUS_WAITING,
waiting_ts: nullNow,
should_init_ts: nullNow,
token: row.token,
retry_attempt: row.retry_count,
settings_id: row.settings_id,
settings: row.settings,
}
newRqList = append(newRqList, queueRow)
row.added_to_queue_ts.Valid = true
row.added_to_queue_ts.Int64 = int64(now)
d.addToAdded(row)
}
err := db.DoInLazyTransaction(func(tx *db.LazyTrx) error {
return addToQueueAndDeleteExpired(tx, newRqList, toDeleteFromWaitingList)
})
if err == nil {
for _, row := range toDeleteFromWaitingList {
d.deletedIds[row.id] = DELETE_IDS_KEEP_GENERATIONS
}
// all rows can expire by TTL in this loop, so check if it the case and notify job generator about it
if len(toDeleteFromWaitingList) > 0 {
d.checkZero("redispatch")
}
if len(newRqList) > 0 {
perHost := make(map[string][]*RunQueueEntry)
for _, row := range newRqList {
perHost[row.hostname] = append(perHost[row.hostname], row)
}
for hostname, rows := range perHost {
notifyAboutNewRQRows(hostname, rows, false)
}
}
return
}
d.tickRedispatchCh = time.After(time.Second)
// restore internal structures back in case of error
log.Warnf("Could not add to run queue for class %s and location %s to database: %s", d.className, d.location, err.Error())
for _, rqRow := range newRqList {
row, ok := d.addedMap[uint64(rqRow.timetable_id.Int64)]
if ok {
row.added_to_queue_ts.Valid = false
row.added_to_queue_ts.Int64 = 0
row.id = uint64(rqRow.timetable_id.Int64)
d.removeFromAdded(row)
d.addToWaiting(row)
} else {
log.Warnf("Internal consistency error: could not find row with timetable id %d", rqRow.timetable_id)
}
}
}
func DebugPrintln(localPrefix, str string) {
log.Debugln(localPrefix, "\n", str)
}