Beispiel #1
0
func batchSetNextGenerateJobTs(tx *db.LazyTrx, className string, params []NextGenParams, settings *ScriptSettings) (err error) {
	if len(params) == 0 {
		return
	}

	fields := `(class_name, location, next_generate_job_ts, jobs_generated_ts, jobs_finished_ts, init_jobs_ts, finish_jobs_ts, generation_id, settings_id)`
	values := make([]string, 0, len(params))

	for _, par := range params {
		firstRun := !par.JobInfo.jobs_generated_ts.Valid
		var prevTs, nextTs uint64

		if !firstRun {
			prevTs = uint64(par.JobInfo.jobs_generated_ts.Int64)
		}

		nextTs = getNextJobGenerateTs(className, firstRun, prevTs, settings)

		values = append(values, fmt.Sprintf("('%s', '%s', FROM_UNIXTIME(%d), NULL, NULL, NULL, NULL, 1, %d)",
			db.EscapeString(className), db.EscapeString(par.Location), nextTs, settings.id))
	}

	_, err = tx.Exec(
		QUERY_BATCH_SET_NEXT_GENERATE_JOB_TS,
		"fields", fields,
		"values", &db.RawData{Data: strings.Join(values, ", ")})
	return
}
Beispiel #2
0
func setRunStatusToInit(tx *db.LazyTrx, run_id uint64, max_time int) (err error) {
	_, err = tx.Exec(
		QUERY_UPDATE_RUN_STATUS_INIT,
		"max_time", max_time,
		"id", run_id)
	return
}
Beispiel #3
0
func setMaxFinishedTs(tx *db.LazyTrx, className string, employeeId int64, ts int64) (err error) {
	_, err = tx.Exec(
		QUERY_SET_MAX_FINISHED_TS_NOW,
		"class_name", className,
		"employee_id", employeeId,
		"ts", ts)
	return
}
Beispiel #4
0
func setNextGenerateJobTs(tx *db.LazyTrx, className string, settings *ScriptSettings) (err error) {
	nextTs := getNextJobGenerateTs(className, true, 0, settings)

	_, err = tx.Exec(
		QUERY_SET_NEXT_GENERATE_JOB_TS,
		"class_name", className,
		"settings_id", settings.id,
		"next_generate_job_ts", nextTs)
	return
}
Beispiel #5
0
func clearJobsResultsForLocations(tx *db.LazyTrx, className string, locations []string) (err error) {
	if len(locations) == 0 {
		return
	}

	_, err = tx.Exec(
		QUERY_CLEAR_JOB_RESULTS_FOR_LOCATIONS,
		"class_name", className,
		"locations", db.INStr(locations))
	return
}
Beispiel #6
0
func setJobsGeneratedTs(tx *db.LazyTrx, className string, locations []string) (err error) {
	if len(locations) == 0 {
		return
	}

	_, err = tx.Exec(
		QUERY_SET_JOBS_GENERATED_TS,
		"class_name", className,
		"locations", db.INStr(locations))
	return
}
Beispiel #7
0
func setInitJobsTs(tx *db.LazyTrx, className string, locations []string) (err error) {
	if len(locations) == 0 {
		return
	}

	_, err = tx.Exec(
		QUERY_SET_INIT_JOBS_TS,
		"class_name", className,
		"locations", db.INStr(locations))
	return
}
Beispiel #8
0
func deleteAddedFromTimetable(tx *db.LazyTrx, ids []uint64) error {
	if len(ids) == 0 {
		return nil
	}

	res, err := tx.Exec(QUERY_DELETE_FROM_TIMETABLE, "ids", db.INUint64(ids), "add_where", " AND added_to_queue_ts IS NOT NULL")
	if err != nil {
		return err
	}
	aff, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if aff != int64(len(ids)) {
		return fmt.Errorf("deleteAddedFromTimetable unexpected ttIds deleted count:%d instead of %d ids:%+v", aff, len(ids), ids)
	}
	return nil
}
Beispiel #9
0
// method inserts rows into timetable and sets insert id
func addToTimetable(tx *db.LazyTrx, ttRows []*TimetableEntry) error {
	if len(ttRows) == 0 {
		return nil
	}

	values := make([]string, 0)

	for _, row := range ttRows {
		val := fmt.Sprintf(
			"('%s', %d, %s, '%s', %d, %s, %d, '%s', '%s', %d, FROM_UNIXTIME(%d), FROM_UNIXTIME(%d))",
			db.EscapeString(row.class_name),
			row.default_retry,
			db.QNullInt64(row.repeat).Data,
			row.method,
			row.finished_successfully,
			db.QNullInt64(row.generation_id).Data,
			row.settings_id,
			db.EscapeString(row.location),
			db.EscapeString(row.JobData),
			0,
			row.created,
			row.NextLaunchTs.Int64)

		values = append(values, val)
	}

	res, err := tx.Exec(QUERY_INSERT_INTO_TIMETABLE, "values", &db.RawData{Data: strings.Join(values, ", ")})
	if err != nil {
		return err
	}

	insId, err := res.LastInsertId()
	if err != nil {
		log.Errorf("Could not get insert id even though insert was successfull: %s", err.Error())
		return err
	}

	for _, row := range ttRows {
		row.id = uint64(insId)
		insId += autoIncrementIncrement
	}

	return nil
}
Beispiel #10
0
func addJobInfo(tx *db.LazyTrx, rows []*JobInfoEntry) (err error) {
	if len(rows) == 0 {
		return
	}

	fields := `(class_name, location, next_generate_job_ts, settings_id)`
	values := make([]string, 0, len(rows))

	for _, row := range rows {
		values = append(values, fmt.Sprintf(
			"('%s', '%s', FROM_UNIXTIME(%d), %d)",
			db.EscapeString(row.class_name), db.EscapeString(row.location), row.next_generate_job_ts.Int64, row.settings_id))
	}

	_, err = tx.Exec(
		QUERY_INSERT_INTO_JOB_INFO,
		"fields", fields,
		"values", &db.RawData{Data: strings.Join(values, ",")})
	return
}
Beispiel #11
0
func deleteFromRunQueue(tx *db.LazyTrx, ids []uint64, prevStatus string) error {
	if len(ids) == 0 {
		return nil
	}

	res, err := tx.Exec(
		QUERY_DELETE_FROM_QUEUE,
		"ids", db.INUint64(ids),
		"status", prevStatus)
	if err != nil {
		return err
	}
	aff, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if aff != int64(len(ids)) {
		return fmt.Errorf("deleteFromRunQueue failed aff:%d expected:%d ids:%+v prevStatus:%s", aff, len(ids), ids, prevStatus)
	}
	return nil
}
Beispiel #12
0
func logTTFinish(tx *db.LazyTrx, row *TimetableEntry, haveOldFinishCount bool, oldFinishCount uint) error {
	var addedToQueueWhere, addedFinishCountWhere string
	if row.added_to_queue_ts.Valid {
		addedToQueueWhere = "AND added_to_queue_ts IS NULL"
	} else {
		addedToQueueWhere = "AND added_to_queue_ts IS NOT NULL"
	}

	if haveOldFinishCount {
		addedFinishCountWhere = fmt.Sprintf("AND finish_count = %d", oldFinishCount)
	} else {
		addedFinishCountWhere = fmt.Sprintf("AND finish_count = %d", row.finish_count-1)
	}

	res, err := tx.Exec(QUERY_UPDATE_TIMETABLE_STATUS,
		"finished_ts", db.QNullInt64(row.finished_ts),
		"next_launch_ts", db.QNullInt64(row.NextLaunchTs),
		"added_to_queue_ts", db.QNullInt64(row.added_to_queue_ts),
		"finish_count", row.finish_count,
		"retry_count", row.retry_count,
		"finished_successfully", row.finished_successfully,
		"id", row.id,
		"add_where", fmt.Sprintf("%s %s", addedToQueueWhere, addedFinishCountWhere))

	if err != nil {
		return err
	}

	aff, err := res.RowsAffected()
	if err != nil {
		return err
	}

	if aff != 1 {
		return fmt.Errorf("unexpected affected rows = %d for tt_row = %+v", aff, row)
	}

	return nil
}
Beispiel #13
0
func updateRunStatus(tx *db.LazyTrx, run_id uint64, status, prevStatus string) error {
	res, err := tx.Exec(
		QUERY_UPDATE_RUN_STATUS,
		"status", status,
		"prev_status", prevStatus,
		"id", run_id)

	if err != nil {
		return err
	}

	aff, err := res.RowsAffected()
	if err != nil {
		return err
	}

	if aff != 1 {
		return fmt.Errorf("Previous status mismatch for rq row id=%d: tried %s -> %s", run_id, prevStatus, status)
	}

	return nil
}
Beispiel #14
0
func doCycle() bool {
	var (
		jiRows         map[string]map[string]*JobInfoEntry
		scripts        map[string]*ScriptEntry
		flags          map[string]*FlagEntry
		scriptsRusage  map[string]*ScriptRusageEntry
		classLocTTRows map[string]map[string][]*TimetableEntry
	)

	unifiedStartTs := time.Now().UnixNano()

	startTs := time.Now().UnixNano()
	err := loadFullState(
		&LoadStateFunc{name: "Scripts", fun: func() (err error) { scripts, err = getGroupedScriptsForPlatform(); return }},
		&LoadStateFunc{name: "JobInfo", fun: func() (err error) { jiRows, err = getGroupedJobInfo(); return }},
		&LoadStateFunc{name: "Flags", fun: func() (err error) { flags, err = getFlags(); return }},
		&LoadStateFunc{name: "ScriptsRusage", fun: func() (err error) { scriptsRusage, err = getScriptRusageStats(); return }},
		&LoadStateFunc{name: "ScriptTimetable", fun: func() (err error) { classLocTTRows, err = selectTimetable(); return }})

	if err != nil {
		log.Errorf("Failed to select state in doCycle: %s", err.Error())
		return false
	}

	log.Debugf("Loaded for %.5f sec", float64(time.Now().UnixNano()-startTs)/1e9)

	startTs = time.Now().UnixNano()
	err = loadSettingsFromRows(jiRows, scripts)
	if err != nil {
		log.Errorf("Could not load settings from rows: %s", err.Error())
		return false
	}

	func() {
		allSettingsMutex.Lock()
		defer allSettingsMutex.Unlock()

		for _, row := range scripts {
			row.settings = allSettings[row.settings_id]
		}
	}()

	scriptsMap.Lock()
	scriptsMap.v = scripts
	scriptsMap.Unlock()

	log.Debugf("  Selected %d rows from flags", len(flags))
	log.Debugf("  Selected %d rows from scripts rusage", len(scriptsRusage))
	log.Debugf("Load settings for %.5f sec", float64(time.Now().UnixNano()-startTs)/1e9)

	startTs = time.Now().UnixNano()

	// We should not try to generate jobs for scripts that are not present in Script table
	// But we should not forget settings (e.g. last generation_id) for that script
	for class_name := range jiRows {
		if _, ok := scripts[class_name]; !ok {
			delete(jiRows, class_name)
		}
	}

	log.Debugf("Selected all for %.5f sec", float64(time.Now().UnixNano()-unifiedStartTs)/1e9)

	startTs = time.Now().UnixNano()
	updateLoadEstimates()

	log.Debugf("Load estimates updated for %.5f sec", float64(time.Now().UnixNano()-startTs)/1e9)
	func() {
		rusageInfo.Lock()
		defer rusageInfo.Unlock()
		log.Debugf("Group hosts: %+v", rusageInfo.groupHosts)
	}()

	startTs = time.Now().UnixNano()

	failedLocationsMutex.Lock()
	failedLocations = make(map[string]bool)
	failedLocationsMutex.Unlock()

	success := true

	if len(scripts) > 0 {
		throttle.setIntervalCh <- time.Second / time.Duration(len(scripts))
	}

	trigger(throttle.c, "throttle, start of cycle")

	for className, script := range scripts {
		<-throttle.c

		tx := new(db.LazyTrx)
		err := tx.Begin()
		if err != nil {
			log.Errorf("Could not start transaction in job generate: %s", err.Error())
			success = false
			continue
		}

		have := make(map[string]bool)
		locTtRows := classLocTTRows[className]
		if locTtRows != nil {
			for rawLoc, v := range locTtRows {
				loc, err := getLocationIdx(script.settings.location_type, rawLoc)
				if err != nil {
					log.Warningf("Broken settings for class %s: %s", className, err.Error())
					loc = rawLoc
				}
				if len(v) > 0 {
					have[loc] = true
				}
			}
		}

		add_to_timetable, err := generateJobs(tx, className, script.settings, jiRows[className], have, flags[className])

		if err != nil {
			log.Errorf("Could generate jobs for class %s: %s", className, err.Error())
			tx.Rollback()
			success = false
			continue
		}

		err = tx.Commit()
		if err != nil {
			log.Errorf("Could not commit generate jobs for class %s: %s", className, err.Error())
			success = false
			continue
		}

		per_location := make(map[string][]*TimetableEntry)

		for _, row := range add_to_timetable {
			allSettingsMutex.Lock()
			row.settings = allSettings[row.settings_id]
			allSettingsMutex.Unlock()

			if row.settings == nil {
				log.Warningf("Internal inconsistency error: Invalid settings for generated row: %+v", row)
				continue
			}

			key := DEFAULT_LOCATION_IDX
			if row.settings.location_type == LOCATION_TYPE_EACH {
				key = row.location
			}

			if _, ok := per_location[key]; !ok {
				per_location[key] = make([]*TimetableEntry, 0)
			}

			per_location[key] = append(per_location[key], row)
		}

		for location, rows := range per_location {
			notifyAboutNewTTRows(className, location, rows, true)
		}
	}

	notifyForFullTTSelect(classLocTTRows, true)

	log.Debugf("Processed %d classes for %.5f sec", len(scripts), float64(time.Now().UnixNano()-startTs)/1e9)
	log.Debugf("Total %.5f sec", float64(time.Now().UnixNano()-unifiedStartTs)/1e9)

	return success
}
Beispiel #15
0
// haveTTRows must be nil if there are no timetable entries for any location
// otherwise it must have only true entries like map["location"] => true
// probably jobs generation can be simplified, it is just the way it is
func generateJobs(tx *db.LazyTrx, className string, settings *ScriptSettings, jiRows map[string]*JobInfoEntry, haveTTRows map[string]bool, flags *FlagEntry) (add_to_timetable []*TimetableEntry, err error) {
	if haveTTRows != nil && len(haveTTRows) == 0 {
		haveTTRows = nil
	}

	now := time.Now().Unix()

	add_to_timetable = make([]*TimetableEntry, 0)
	add_job_info := make([]*JobInfoEntry, 0)
	set_finish_jobs := make([]string, 0)
	set_init_jobs := make([]string, 0)
	set_jobs_generated_js := make([]string, 0)
	prepare_next_generation := make([]NextGenParams, 0)

	have_finish_jobs := settings.jobs.Have_finish_jobs
	is_any := (settings.location_type == LOCATION_TYPE_ANY)
	is_temporary := settings.jobs.Temporary
	temporary_can_run := false

	if flags != nil {
		if flags.kill_requested_ts.Valid {
			is_done := (haveTTRows == nil)
			if is_done {
				log.Printf("Class %s is done, all is ok", className)

				if !flags.killed_ts.Valid {
					tx.AddCommitCallback(func() { continueDispatchAfterKill(className) })
					if err = setKilledFlag(tx, className); err != nil {
						return
					}

					if err = prepareNextGeneration(tx, have_finish_jobs, className, settings); err != nil {
						return
					}
				}
			} else {
				log.Printf("Class %s is not done", className)

				startKilling(className)

				// not the best place to put it, but it works
				if err = setMaxFinishedTs(tx, className, flags.kill_request_employee_id.Int64, flags.kill_requested_ts.Int64); err != nil {
					return
				}
			}

			return
		}

		// Stop generating new job generations when we are on pause
		if flags.pause_requested_ts.Valid {
			is_done := generationFinished(className, haveTTRows, jiRows, settings)

			if is_done && !flags.paused_ts.Valid {
				if err = setPausedFlag(tx, className); err != nil {
					return
				}

				flags.paused_ts = sql.NullInt64{Int64: now, Valid: true}
			}

			if !is_any || flags.paused_ts.Valid {
				return
			}
		}

		if is_temporary && flags.run_requested_ts.Valid && is_any {
			// We accepted run request, which means that we already generated jobs
			if flags.run_accepted_ts.Valid {
				if generationFinished(className, haveTTRows, jiRows, settings) {
					if err = resetRunRequest(tx, className); err != nil {
						return
					}

					if err = prepareNextGeneration(tx, have_finish_jobs, className, settings); err != nil {
						return
					}

					return
				}
			} else {
				if err = setRunAccepted(tx, className); err != nil {
					return
				}
			}

			temporary_can_run = true
		}
	}

	if is_temporary && !temporary_can_run || settings.jobs.Type == JOBS_TYPE_NONE {
		return
	}

	locations := make([]string, 0)

	if !is_any {
		all_locations := getLocations(settings)
		timetable_locations := make(map[string]bool)

		if haveTTRows != nil {
			for location, _ := range haveTTRows {
				timetable_locations[location] = true
			}
		}

		// there can be failed hosts that are still running: we must really compare host names, not just counts
		for _, loc := range all_locations {
			if _, ok := timetable_locations[loc]; !ok {
				locations = append(locations, loc)
			}
		}

		if len(locations) == 0 {
			return
		}
	} else {
		if haveTTRows != nil && len(haveTTRows) > 0 {
			return
		}

		locations = getLocations(settings)
	}

	tt_location_type := LOCATION_TYPE_EACH
	if is_any {
		tt_location_type = LOCATION_TYPE_ANY
	}

	for _, location := range locations {
		job_info_key, gliErr := getLocationIdx(tt_location_type, location)
		if gliErr != nil {
			log.Warningf("Error getting location index for %s for location_type %s and location %s: %s", className, tt_location_type, location, gliErr.Error())
			continue
		}

		var row *JobInfoEntry

		if jiRows == nil || jiRows[job_info_key] == nil {
			row = &JobInfoEntry{generation_id: 0,
				class_name:           className,
				location:             job_info_key,
				next_generate_job_ts: sql.NullInt64{Int64: int64(getNextJobGenerateTs(className, true, 0, settings)), Valid: true},
				settings_id:          settings.id}

			add_job_info = append(add_job_info, row)
		} else {
			row = jiRows[job_info_key]
		}

		tt_row := &TimetableEntry{
			class_name:            className,
			default_retry:         settings.retry_job,
			repeat:                settings.repeat_job,
			method:                METHOD_RUN,
			finished_successfully: 0,
			generation_id:         sql.NullInt64{Int64: int64(row.generation_id), Valid: true},
			settings_id:           row.settings_id,
			location:              location,
			created:               uint64(now),
		}

		tt_row.NextLaunchTs.Valid = true
		tt_row.NextLaunchTs.Int64 = now

		if row.jobs_generated_ts.Valid || row.init_jobs_ts.Valid {
			if have_finish_jobs && !row.finish_jobs_ts.Valid {
				set_finish_jobs = append(set_finish_jobs, job_info_key)

				tt_row.JobData = `"finishJobs"`
				tt_row.method = METHOD_FINISH_JOBS
				tt_row.default_retry = settings.retry_job

				add_to_timetable = append(add_to_timetable, tt_row)
			} else {
				prepare_next_generation = append(prepare_next_generation, NextGenParams{Location: job_info_key, JobInfo: row})
			}

			continue
		} else if row.next_generate_job_ts.Int64 > now {
			continue
		}

		if settings.jobs.Type == JOBS_TYPE_CUSTOM {
			set_init_jobs = append(set_init_jobs, job_info_key)

			tt_row.JobData = `"initJobs"`
			tt_row.method = METHOD_INIT_JOBS
			tt_row.default_retry = uint32(settings.retry.Int64)

			add_to_timetable = append(add_to_timetable, tt_row)
			continue
		}

		jobs, mjlErr := makeJobsList(settings.jobs, settings.instance_count, className)
		if mjlErr != nil {
			log.Warningf("Error generating jobs for %+v with instance_count=%d and jobs=%s: %s", className, settings.instance_count, settings.jobs, mjlErr.Error())
			continue
		}

		for _, job := range jobs {
			tt_row_copy := new(TimetableEntry)
			*tt_row_copy = *tt_row
			tt_row_copy.JobData = job
			add_to_timetable = append(add_to_timetable, tt_row_copy)
		}

		set_jobs_generated_js = append(set_jobs_generated_js, job_info_key)
	}

	if err = addJobInfo(tx, add_job_info); err != nil {
		return
	}

	if err = setFinishJobsTs(tx, className, set_finish_jobs); err != nil {
		return
	}

	if err = batchPrepareNextGeneration(tx, have_finish_jobs, className, prepare_next_generation, settings); err != nil {
		return
	}

	if err = setInitJobsTs(tx, className, set_init_jobs); err != nil {
		return
	}

	if err = setJobsGeneratedTs(tx, className, set_jobs_generated_js); err != nil {
		return
	}

	if err = addToTimetable(tx, add_to_timetable); err != nil {
		return
	}

	return
}
Beispiel #16
0
func setPausedFlag(tx *db.LazyTrx, className string) (err error) {
	_, err = tx.Exec(
		`UPDATE `+TABLE_SCRIPT_FLAGS+` SET paused_ts = NOW() WHERE class_name = '#class_name#'`,
		"class_name", className)
	return
}
Beispiel #17
0
func setRunAccepted(tx *db.LazyTrx, className string) (err error) {
	_, err = tx.Exec(QUERY_SET_RUN_ACCEPTED, "class_name", className)
	return
}
Beispiel #18
0
func resetRunRequest(tx *db.LazyTrx, className string) (err error) {
	_, err = tx.Exec(QUERY_RESET_RUN_REQUEST, "class_name", className)
	return
}
Beispiel #19
0
func clearJobsResults(tx *db.LazyTrx, className string) (err error) {
	_, err = tx.Exec(
		QUERY_CLEAR_JOB_RESULTS,
		"class_name", className)
	return
}
Beispiel #20
0
func addToQueueAndDeleteExpired(tx *db.LazyTrx, rows []*RunQueueEntry, toDelete []*TimetableEntry) error {
	if len(rows) > 0 {
		values := make([]string, 0, len(rows))
		ttIds := make([]uint64, 0, len(rows))

		fields := `(class_name, timetable_id, generation_id, hostname, hostname_idx, job_data, method, created, run_status, waiting_ts, should_init_ts, token, retry_attempt, settings_id)`

		for _, row := range rows {
			ttIds = append(ttIds, uint64(row.timetable_id.Int64))

			val := fmt.Sprint(
				"('"+db.EscapeString(row.ClassName)+"',",
				db.QNullInt64(row.timetable_id).Data, ",",
				row.generation_id.Int64, ",",
				"'"+db.EscapeString(row.hostname)+"',",
				row.hostname_idx, ",",
				"'"+db.EscapeString(row.JobData)+"',",
				"'"+db.EscapeString(row.method)+"',",
				"FROM_UNIXTIME(", row.created.Int64, "),",
				"'"+db.EscapeString(row.RunStatus)+"',",
				"FROM_UNIXTIME(", row.waiting_ts.Int64, "),",
				"FROM_UNIXTIME(", row.should_init_ts.Int64, "),",
				"'"+db.EscapeString(row.token), "',",
				row.retry_attempt, ",",
				row.settings_id, ")")

			values = append(values, val)
		}

		res, err := tx.Exec(QUERY_INSERT_INTO_RUN_QUEUE, "fields", fields, "values", &db.RawData{Data: strings.Join(values, ",")})
		if err != nil {
			return err
		}

		insId, err := res.LastInsertId()
		if err != nil {
			return err
		}

		for _, row := range rows {
			row.Id = uint64(insId)
			insId += autoIncrementIncrement
		}

		sort.Sort(common.UInt64Slice(ttIds))

		res, err = tx.Exec(QUERY_LOG_ADD_TO_QUEUE, "ids", db.INUint64(ttIds))
		if err != nil {
			return err
		}
		aff, err := res.RowsAffected()
		if err != nil {
			return err
		}
		if aff != int64(len(ttIds)) {
			return fmt.Errorf("update ur cache bro aff: %d ttIds:%+v", aff, ttIds)
		}
	}

	if len(toDelete) > 0 {
		ttIds := make([]uint64, 0, len(toDelete))
		for _, row := range toDelete {
			ttIds = append(ttIds, row.id)
		}

		sort.Sort(common.UInt64Slice(ttIds))

		res, err := tx.Exec(QUERY_DELETE_FROM_TIMETABLE, "ids", db.INUint64(ttIds), "add_where", " AND added_to_queue_ts IS NULL")
		if err != nil {
			return err
		}
		aff, err := res.RowsAffected()
		if err != nil {
			return err
		}
		if aff != int64(len(ttIds)) {
			return fmt.Errorf("addToQueueAndDeleteExpired unexpected ttIds deleted count:%d instead of %d ids:%+v", aff, len(ttIds), ttIds)
		}
	}

	return nil
}