// getWorkFromSpec forcibly retrieves a work unit from a work spec.
// It could create a work unit if spec is a continuous spec with no
// available units. It ignores other constraints, such as whether the
// work spec is paused.
func (w *worker) getWorkFromSpec(spec *workSpec, meta *coordinate.WorkSpecMeta) *attempt {
var unit *workUnit
now := w.Coordinate().clock.Now()
if len(spec.available) != 0 {
unit = spec.available.Next()
} else if meta.CanStartContinuous(now) {
// Make a brand new work unit. Its key is the string
// form of a time_t.
seconds := now.Unix()
nano := now.Nanosecond()
milli := nano / 1000000
name := fmt.Sprintf("%d.%03d", seconds, milli)
var exists bool
unit, exists = spec.workUnits[name]
if !exists {
unit = &workUnit{
name: name,
data: map[string]interface{}{},
workSpec: spec,
}
spec.workUnits[name] = unit
}
spec.meta.NextContinuous = now.Add(meta.Interval)
} else {
return nil
}
return w.makeAttempt(unit, time.Duration(0))
}
func (spec *workSpec) SetMeta(meta coordinate.WorkSpecMeta) error {
return spec.do(func() error {
// Preserve immutable fields (taking advantage of meta pass-by-value)
meta.CanBeContinuous = spec.meta.CanBeContinuous
meta.NextWorkSpecName = spec.meta.NextWorkSpecName
meta.Runtime = spec.meta.Runtime
// If this cannot be continuous, force-clear that flag
if !meta.CanBeContinuous {
meta.Continuous = false
}
spec.meta = meta
return nil
})
}
// TestMetaContinuous specifically checks that you cannot enable the
// "continuous" flag on non-continuous work specs.
func TestMetaContinuous(t *testing.T) {
var (
err error
namespace coordinate.Namespace
spec coordinate.WorkSpec
meta coordinate.WorkSpecMeta
)
namespace, err = Coordinate.Namespace("TestMetaContinuous")
if !assert.NoError(t, err) {
return
}
defer namespace.Destroy()
spec, err = namespace.SetWorkSpec(map[string]interface{}{
"name": "spec",
"min_gb": 1,
})
if !assert.NoError(t, err) {
return
}
meta, err = spec.Meta(false)
if assert.NoError(t, err) {
assert.False(t, meta.Continuous)
assert.False(t, meta.CanBeContinuous)
}
meta.Continuous = true
err = spec.SetMeta(meta)
assert.NoError(t, err)
meta, err = spec.Meta(false)
if assert.NoError(t, err) {
// Cannot set the "continuous" flag
assert.False(t, meta.Continuous)
assert.False(t, meta.CanBeContinuous)
}
}
// ControlWorkSpec makes changes to a work spec that are not directly
// reflected in the work spec definition. This allows work specs to
// be paused or to stop generating new continuous jobs.
// ControlWorkSpecOptions has a complete listing of what can be done.
func (jobs *JobServer) ControlWorkSpec(workSpecName string, options map[string]interface{}) (bool, string, error) {
var (
cwsOptions ControlWorkSpecOptions
decoder *mapstructure.Decoder
err error
metadata mapstructure.Metadata
workSpec coordinate.WorkSpec
wsMeta coordinate.WorkSpecMeta
)
workSpec, err = jobs.Namespace.WorkSpec(workSpecName)
if err == nil {
// We care a lot about "false" vs. not present for
// these things. Manually create the decoder.
config := mapstructure.DecoderConfig{
Result: &cwsOptions,
Metadata: &metadata,
}
decoder, err = mapstructure.NewDecoder(&config)
}
if err == nil {
err = decoder.Decode(options)
}
// Get the existing metadata, then change it based on what
// we got provided
if err == nil {
wsMeta, err = workSpec.Meta(false)
}
if err == nil {
for _, key := range metadata.Keys {
switch key {
case "Continuous":
wsMeta.Continuous = cwsOptions.Continuous
case "Status":
wsMeta.Paused = cwsOptions.Status == Paused
case "Weight":
wsMeta.Weight = cwsOptions.Weight
case "Interval":
wsMeta.Interval = time.Duration(cwsOptions.Interval) * time.Second
case "MaxRunning":
wsMeta.MaxRunning = cwsOptions.MaxRunning
}
}
}
if err == nil {
err = workSpec.SetMeta(wsMeta)
}
return err == nil, "", err
}
func (w *worker) requestAttemptsForSpec(req coordinate.AttemptRequest, spec *workSpec, meta *coordinate.WorkSpecMeta) ([]coordinate.Attempt, error) {
var (
attempts []coordinate.Attempt
count int
err error
)
// Adjust the work unit count based on what's possible here
count = req.NumberOfWorkUnits
if count < 1 {
count = 1
}
if meta.MaxAttemptsReturned > 0 && count > meta.MaxAttemptsReturned {
count = meta.MaxAttemptsReturned
}
if meta.MaxRunning > 0 && count > meta.MaxRunning-meta.PendingCount {
count = meta.MaxRunning - meta.PendingCount
}
continuous := false
length := time.Duration(15) * time.Minute
err = withTx(w, false, func(tx *sql.Tx) error {
var err error
now := w.Coordinate().clock.Now()
// Take an advisory lock for the work spec. It doesn't
// make sense for multiple concurrent actors to progress
// beyond this point (they will hit the same work units
// in chooseAndMakeAttempts() and one will roll back) and
// this reduces the database load. We are still protected
// by standard SQL transactionality.
params := queryParams{}
query := "SELECT pg_advisory_xact_lock(0, " + params.Param(spec.id) + ")"
_, err = tx.Exec(query, params...)
if err != nil {
return err
}
// Try to create attempts from pre-existing work units
// (assuming we expect there to be some)
if meta.AvailableCount > 0 {
attempts, err = w.chooseAndMakeAttempts(tx, spec, count, now, length)
}
if err != nil || len(attempts) > 0 {
return err
}
// If there were none, but the selected work spec is
// continuous, maybe we can create a work unit and an
// attempt
if meta.CanStartContinuous(now) {
var unit *workUnit
var attempt *attempt
continuous = true
unit, err = w.createContinuousUnit(tx, spec, meta, now)
if err == nil && unit != nil {
attempt, err = makeAttempt(tx, unit, w, length)
}
if err == nil && attempt != nil {
attempts = []coordinate.Attempt{attempt}
}
}
// Whatever happened, end of the road
return err
})
// On a very bad day, we could have gone to create continuous
// work units, but an outright INSERT failed from a duplicate
// key. If this happened, just pretend we didn't actually get
// any attempts back, which will trigger the retry loop in our
// caller.
if continuous && err != nil && isDuplicateUnitName(err) {
attempts = nil
err = nil
}
return attempts, err
}
// AllMetas retrieves the metadata for all work specs. This is
// expected to run within a pre-existing transaction. On success,
// returns maps from work spec name to work spec object and to
// metadata object.
func (ns *namespace) allMetas(tx *sql.Tx, withCounts bool) (map[string]*workSpec, map[string]*coordinate.WorkSpecMeta, error) {
params := queryParams{}
query := buildSelect([]string{
workSpecID,
workSpecName,
workSpecPriority,
workSpecWeight,
workSpecPaused,
workSpecContinuous,
workSpecCanBeContinuous,
workSpecMinMemoryGb,
workSpecInterval,
workSpecNextContinuous,
workSpecMaxRunning,
workSpecMaxAttemptsReturned,
workSpecNextWorkSpec,
workSpecRuntime,
}, []string{
workSpecTable,
}, []string{
workSpecInNamespace(¶ms, ns.id),
})
rows, err := tx.Query(query, params...)
if err != nil {
return nil, nil, err
}
specs := make(map[string]*workSpec)
metas := make(map[string]*coordinate.WorkSpecMeta)
err = scanRows(rows, func() error {
var (
spec workSpec
meta coordinate.WorkSpecMeta
interval string
nextContinuous pq.NullTime
err error
)
err = rows.Scan(&spec.id, &spec.name, &meta.Priority,
&meta.Weight, &meta.Paused, &meta.Continuous,
&meta.CanBeContinuous, &meta.MinMemoryGb,
&interval, &nextContinuous, &meta.MaxRunning,
&meta.MaxAttemptsReturned,
&meta.NextWorkSpecName, &meta.Runtime)
if err != nil {
return err
}
spec.namespace = ns
meta.NextContinuous = nullTimeToTime(nextContinuous)
meta.Interval, err = sqlToDuration(interval)
if err != nil {
return err
}
specs[spec.name] = &spec
metas[spec.name] = &meta
return nil
})
if err != nil {
return nil, nil, err
}
if withCounts {
// A single query that selects both "available" and
// "pending" is hopelessly expensive. Also, in the
// only place this is called (in RequestAttempts) we
// need to know whether or not there are any available
// attempts, but we don't really care how many there
// are so long as there are more than zero.
//
// Pending:
params = queryParams{}
query = buildSelect([]string{workSpecName, "COUNT(*)"},
[]string{workSpecTable, attemptTable},
[]string{
workSpecInNamespace(¶ms, ns.id),
attemptInThisSpec,
attemptIsPending,
})
query += " GROUP BY " + workSpecName
rows, err = tx.Query(query, params...)
if err != nil {
return nil, nil, err
}
err = scanRows(rows, func() error {
var name string
var count int
err := rows.Scan(&name, &count)
if err == nil {
metas[name].PendingCount = count
}
return err
})
// Available count (0/1):
now := ns.Coordinate().clock.Now()
params = queryParams{}
query = buildSelect([]string{
workUnitSpec,
}, []string{
workUnitTable,
}, []string{
workUnitHasNoAttempt,
"NOT " + workUnitTooSoon(¶ms, now),
})
query = buildSelect([]string{
workSpecName,
}, []string{
workSpecTable,
}, []string{
workSpecInNamespace(¶ms, ns.id),
workSpecID + " IN (" + query + ")",
})
rows, err = tx.Query(query, params...)
err = scanRows(rows, func() error {
var name string
err := rows.Scan(&name)
if err == nil {
metas[name].AvailableCount = 1
}
return err
})
if err != nil {
return nil, nil, err
}
}
return specs, metas, nil
}
func (spec *workSpec) Meta(withCounts bool) (coordinate.WorkSpecMeta, error) {
// If we need counts, we need to run expiry so that the
// available/pending counts are rightish
if withCounts {
spec.Coordinate().Expiry.Do(spec)
}
var meta coordinate.WorkSpecMeta
err := withTx(spec, true, func(tx *sql.Tx) error {
var (
params queryParams
query string
interval string
nextContinuous pq.NullTime
)
query = buildSelect([]string{
workSpecPriority,
workSpecWeight,
workSpecPaused,
workSpecContinuous,
workSpecCanBeContinuous,
workSpecMinMemoryGb,
workSpecInterval,
workSpecNextContinuous,
workSpecMaxRunning,
workSpecMaxAttemptsReturned,
workSpecNextWorkSpec,
workSpecRuntime,
}, []string{
workSpecTable,
}, []string{
isWorkSpec(¶ms, spec.id),
})
row := tx.QueryRow(query, params...)
err := row.Scan(
&meta.Priority,
&meta.Weight,
&meta.Paused,
&meta.Continuous,
&meta.CanBeContinuous,
&meta.MinMemoryGb,
&interval,
&nextContinuous,
&meta.MaxRunning,
&meta.MaxAttemptsReturned,
&meta.NextWorkSpecName,
&meta.Runtime,
)
if err == sql.ErrNoRows {
return coordinate.ErrGone
}
if err != nil {
return err
}
meta.NextContinuous = nullTimeToTime(nextContinuous)
meta.Interval, err = sqlToDuration(interval)
if err != nil {
return err
}
// Find counts with a second query, if requested
if !withCounts {
return nil
}
params = queryParams{}
query = buildSelect([]string{
attemptStatus,
"COUNT(*)",
}, []string{
workUnitAttemptJoin,
}, []string{
workUnitInSpec(¶ms, spec.id),
})
query += " GROUP BY " + attemptStatus
rows, err := tx.Query(query, params...)
if err != nil {
return err
}
return scanRows(rows, func() error {
var status sql.NullString
var count int
err := rows.Scan(&status, &count)
if err != nil {
return err
}
if !status.Valid {
meta.AvailableCount += count
} else {
switch status.String {
case "expired":
meta.AvailableCount += count
case "retryable":
meta.AvailableCount += count
case "pending":
meta.PendingCount += count
}
}
return nil
})
})
return meta, err
}