// haveTTRows must be nil if there are no timetable entries for any location // otherwise it must have only true entries like map["location"] => true // probably jobs generation can be simplified, it is just the way it is func generateJobs(tx *db.LazyTrx, className string, settings *ScriptSettings, jiRows map[string]*JobInfoEntry, haveTTRows map[string]bool, flags *FlagEntry) (add_to_timetable []*TimetableEntry, err error) { if haveTTRows != nil && len(haveTTRows) == 0 { haveTTRows = nil } now := time.Now().Unix() add_to_timetable = make([]*TimetableEntry, 0) add_job_info := make([]*JobInfoEntry, 0) set_finish_jobs := make([]string, 0) set_init_jobs := make([]string, 0) set_jobs_generated_js := make([]string, 0) prepare_next_generation := make([]NextGenParams, 0) have_finish_jobs := settings.jobs.Have_finish_jobs is_any := (settings.location_type == LOCATION_TYPE_ANY) is_temporary := settings.jobs.Temporary temporary_can_run := false if flags != nil { if flags.kill_requested_ts.Valid { is_done := (haveTTRows == nil) if is_done { log.Printf("Class %s is done, all is ok", className) if !flags.killed_ts.Valid { tx.AddCommitCallback(func() { continueDispatchAfterKill(className) }) if err = setKilledFlag(tx, className); err != nil { return } if err = prepareNextGeneration(tx, have_finish_jobs, className, settings); err != nil { return } } } else { log.Printf("Class %s is not done", className) startKilling(className) // not the best place to put it, but it works if err = setMaxFinishedTs(tx, className, flags.kill_request_employee_id.Int64, flags.kill_requested_ts.Int64); err != nil { return } } return } // Stop generating new job generations when we are on pause if flags.pause_requested_ts.Valid { is_done := generationFinished(className, haveTTRows, jiRows, settings) if is_done && !flags.paused_ts.Valid { if err = setPausedFlag(tx, className); err != nil { return } flags.paused_ts = sql.NullInt64{Int64: now, Valid: true} } if !is_any || flags.paused_ts.Valid { return } } if is_temporary && flags.run_requested_ts.Valid && is_any { // We accepted run request, which means that we already generated jobs if flags.run_accepted_ts.Valid { if generationFinished(className, haveTTRows, jiRows, settings) { if err = resetRunRequest(tx, className); err != nil { return } if err = prepareNextGeneration(tx, have_finish_jobs, className, settings); err != nil { return } return } } else { if err = setRunAccepted(tx, className); err != nil { return } } temporary_can_run = true } } if is_temporary && !temporary_can_run || settings.jobs.Type == JOBS_TYPE_NONE { return } locations := make([]string, 0) if !is_any { all_locations := getLocations(settings) timetable_locations := make(map[string]bool) if haveTTRows != nil { for location, _ := range haveTTRows { timetable_locations[location] = true } } // there can be failed hosts that are still running: we must really compare host names, not just counts for _, loc := range all_locations { if _, ok := timetable_locations[loc]; !ok { locations = append(locations, loc) } } if len(locations) == 0 { return } } else { if haveTTRows != nil && len(haveTTRows) > 0 { return } locations = getLocations(settings) } tt_location_type := LOCATION_TYPE_EACH if is_any { tt_location_type = LOCATION_TYPE_ANY } for _, location := range locations { job_info_key, gliErr := getLocationIdx(tt_location_type, location) if gliErr != nil { log.Warningf("Error getting location index for %s for location_type %s and location %s: %s", className, tt_location_type, location, gliErr.Error()) continue } var row *JobInfoEntry if jiRows == nil || jiRows[job_info_key] == nil { row = &JobInfoEntry{generation_id: 0, class_name: className, location: job_info_key, next_generate_job_ts: sql.NullInt64{Int64: int64(getNextJobGenerateTs(className, true, 0, settings)), Valid: true}, settings_id: settings.id} add_job_info = append(add_job_info, row) } else { row = jiRows[job_info_key] } tt_row := &TimetableEntry{ class_name: className, default_retry: settings.retry_job, repeat: settings.repeat_job, method: METHOD_RUN, finished_successfully: 0, generation_id: sql.NullInt64{Int64: int64(row.generation_id), Valid: true}, settings_id: row.settings_id, location: location, created: uint64(now), } tt_row.NextLaunchTs.Valid = true tt_row.NextLaunchTs.Int64 = now if row.jobs_generated_ts.Valid || row.init_jobs_ts.Valid { if have_finish_jobs && !row.finish_jobs_ts.Valid { set_finish_jobs = append(set_finish_jobs, job_info_key) tt_row.JobData = `"finishJobs"` tt_row.method = METHOD_FINISH_JOBS tt_row.default_retry = settings.retry_job add_to_timetable = append(add_to_timetable, tt_row) } else { prepare_next_generation = append(prepare_next_generation, NextGenParams{Location: job_info_key, JobInfo: row}) } continue } else if row.next_generate_job_ts.Int64 > now { continue } if settings.jobs.Type == JOBS_TYPE_CUSTOM { set_init_jobs = append(set_init_jobs, job_info_key) tt_row.JobData = `"initJobs"` tt_row.method = METHOD_INIT_JOBS tt_row.default_retry = uint32(settings.retry.Int64) add_to_timetable = append(add_to_timetable, tt_row) continue } jobs, mjlErr := makeJobsList(settings.jobs, settings.instance_count, className) if mjlErr != nil { log.Warningf("Error generating jobs for %+v with instance_count=%d and jobs=%s: %s", className, settings.instance_count, settings.jobs, mjlErr.Error()) continue } for _, job := range jobs { tt_row_copy := new(TimetableEntry) *tt_row_copy = *tt_row tt_row_copy.JobData = job add_to_timetable = append(add_to_timetable, tt_row_copy) } set_jobs_generated_js = append(set_jobs_generated_js, job_info_key) } if err = addJobInfo(tx, add_job_info); err != nil { return } if err = setFinishJobsTs(tx, className, set_finish_jobs); err != nil { return } if err = batchPrepareNextGeneration(tx, have_finish_jobs, className, prepare_next_generation, settings); err != nil { return } if err = setInitJobsTs(tx, className, set_init_jobs); err != nil { return } if err = setJobsGeneratedTs(tx, className, set_jobs_generated_js); err != nil { return } if err = addToTimetable(tx, add_to_timetable); err != nil { return } return }