// TestPrefilledMeta tests that WorkSpec.Meta() fills in correctly from
// "magic" keys in a work spec.
func (s *Suite) TestPrefilledMeta(c *check.C) {
var (
err error
spec coordinate.WorkSpec
meta coordinate.WorkSpecMeta
)
spec, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "spec",
"min_gb": 1,
"priority": 10,
"weight": 100,
"disabled": true,
"continuous": true,
"interval": 60,
"max_running": 10,
"max_getwork": 1,
"then": "spec2",
})
c.Assert(err, check.IsNil)
meta, err = spec.Meta(false)
c.Assert(err, check.IsNil)
c.Check(meta.Priority, check.Equals, 10)
c.Check(meta.Weight, check.Equals, 100)
c.Check(meta.Paused, check.Equals, true)
c.Check(meta.Continuous, check.Equals, true)
c.Check(meta.CanBeContinuous, check.Equals, true)
c.Check(meta.Interval, check.Equals, time.Duration(60)*time.Second)
c.Check(meta.NextContinuous, check.Equals, time.Time{})
c.Check(meta.MaxRunning, check.Equals, 10)
c.Check(meta.MaxAttemptsReturned, check.Equals, 1)
c.Check(meta.NextWorkSpecName, check.Equals, "spec2")
c.Check(meta.AvailableCount, check.Equals, 0)
c.Check(meta.PendingCount, check.Equals, 0)
}
// TestDefaultMeta tests that WorkSpec.Meta gets the correct defaults,
// which in a couple of cases are not zero values.
func (s *Suite) TestDefaultMeta(c *check.C) {
var (
err error
spec coordinate.WorkSpec
meta coordinate.WorkSpecMeta
)
spec, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "spec",
"min_gb": 1,
})
c.Assert(err, check.IsNil)
meta, err = spec.Meta(false)
c.Assert(err, check.IsNil)
c.Check(meta.Priority, check.Equals, 0)
c.Check(meta.Weight, check.Equals, 20)
c.Check(meta.Paused, check.Equals, false)
c.Check(meta.Continuous, check.Equals, false)
c.Check(meta.CanBeContinuous, check.Equals, false)
c.Check(meta.Interval, check.Equals, time.Duration(0))
c.Check(meta.NextContinuous, check.Equals, time.Time{})
c.Check(meta.MaxRunning, check.Equals, 0)
c.Check(meta.MaxAttemptsReturned, check.Equals, 0)
c.Check(meta.NextWorkSpecName, check.Equals, "")
c.Check(meta.AvailableCount, check.Equals, 0)
c.Check(meta.PendingCount, check.Equals, 0)
}
// PrioritizeWorkUnits changes the priorities of some number of work
// units. The actual work units are in options["work_unit_keys"]. A
// higher priority results in the work units being scheduled sooner.
func (jobs *JobServer) PrioritizeWorkUnits(workSpecName string, options map[string]interface{}) (bool, string, error) {
var (
err error
query coordinate.WorkUnitQuery
workSpec coordinate.WorkSpec
)
pwuOptions := PrioritizeWorkUnitsOptions{
Priority: math.NaN(),
Adjustment: math.NaN(),
}
workSpec, err = jobs.Namespace.WorkSpec(workSpecName)
if err == nil {
err = decode(&pwuOptions, options)
}
if err == nil && pwuOptions.WorkUnitKeys == nil {
return false, "missing work_unit_keys", err
}
if err == nil {
query.Names = pwuOptions.WorkUnitKeys
if !math.IsNaN(pwuOptions.Priority) {
err = workSpec.SetWorkUnitPriorities(query, pwuOptions.Priority)
} else if !math.IsNaN(pwuOptions.Adjustment) {
err = workSpec.AdjustWorkUnitPriorities(query, pwuOptions.Adjustment)
}
}
return err == nil, "", err
}
// ControlWorkSpec makes changes to a work spec that are not directly
// reflected in the work spec definition. This allows work specs to
// be paused or to stop generating new continuous jobs.
// ControlWorkSpecOptions has a complete listing of what can be done.
func (jobs *JobServer) ControlWorkSpec(workSpecName string, options map[string]interface{}) (bool, string, error) {
var (
cwsOptions ControlWorkSpecOptions
decoder *mapstructure.Decoder
err error
metadata mapstructure.Metadata
workSpec coordinate.WorkSpec
wsMeta coordinate.WorkSpecMeta
)
workSpec, err = jobs.Namespace.WorkSpec(workSpecName)
if err == nil {
// We care a lot about "false" vs. not present for
// these things. Manually create the decoder.
config := mapstructure.DecoderConfig{
Result: &cwsOptions,
Metadata: &metadata,
}
decoder, err = mapstructure.NewDecoder(&config)
}
if err == nil {
err = decoder.Decode(options)
}
// Get the existing metadata, then change it based on what
// we got provided
if err == nil {
wsMeta, err = workSpec.Meta(false)
}
if err == nil {
for _, key := range metadata.Keys {
switch key {
case "Continuous":
wsMeta.Continuous = cwsOptions.Continuous
case "Status":
wsMeta.Paused = cwsOptions.Status == Paused
case "Weight":
wsMeta.Weight = cwsOptions.Weight
case "Interval":
wsMeta.Interval = time.Duration(cwsOptions.Interval) * time.Second
case "MaxRunning":
wsMeta.MaxRunning = cwsOptions.MaxRunning
}
}
}
if err == nil {
err = workSpec.SetMeta(wsMeta)
}
return err == nil, "", err
}
// TestMetaContinuous specifically checks that you cannot enable the
// "continuous" flag on non-continuous work specs.
func (s *Suite) TestMetaContinuous(c *check.C) {
var (
err error
spec coordinate.WorkSpec
meta coordinate.WorkSpecMeta
)
// ...also...
spec, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "spec",
"min_gb": 1,
})
c.Assert(err, check.IsNil)
meta, err = spec.Meta(false)
c.Assert(err, check.IsNil)
c.Check(meta.Continuous, check.Equals, false)
c.Check(meta.CanBeContinuous, check.Equals, false)
meta.Continuous = true
err = spec.SetMeta(meta)
c.Assert(err, check.IsNil)
meta, err = spec.Meta(false)
c.Assert(err, check.IsNil)
// Cannot set the "continuous" flag
c.Check(meta.Continuous, check.Equals, false)
c.Check(meta.CanBeContinuous, check.Equals, false)
}
// makeWorkUnits creates a handful of work units within a work spec.
// These have keys "available", "pending", "finished", "failed",
// "expired", and "retryable", and wind up in the corresponding
// states.
func makeWorkUnits(spec coordinate.WorkSpec, worker coordinate.Worker) (map[string]coordinate.WorkUnit, error) {
result := map[string]coordinate.WorkUnit{
"available": nil,
"pending": nil,
"finished": nil,
"failed": nil,
"expired": nil,
"retryable": nil,
}
for key := range result {
unit, err := spec.AddWorkUnit(key, map[string]interface{}{}, 0)
if err != nil {
return nil, err
}
result[key] = unit
// Run the workflow
if key == "available" {
continue
}
attempt, err := worker.MakeAttempt(unit, time.Duration(0))
if err != nil {
return nil, err
}
switch key {
case "pending":
{
} // leave it running
case "finished":
err = attempt.Finish(nil)
case "failed":
err = attempt.Fail(nil)
case "expired":
err = attempt.Expire(nil)
case "retryable":
err = attempt.Retry(nil)
}
if err != nil {
return nil, err
}
}
return result, nil
}
func getWorkSpecData(spec coordinate.WorkSpec) (map[string]interface{}, error) {
data, err := spec.Data()
if err != nil {
return nil, err
}
byteify := func(key string) {
value, present := data[key]
if !present {
return
}
string, ok := value.(string)
if !ok {
return
}
data[key] = []byte(string)
}
byteify("module")
byteify("run_function")
return data, nil
}
// TestSpecCreateDestroy performs basic work spec lifetime tests.
func (s *Suite) TestSpecCreateDestroy(c *check.C) {
var (
data map[string]interface{}
dict map[string]interface{}
spec coordinate.WorkSpec
name string
names []string
err error
)
name = "spec"
spec, err = s.Namespace.WorkSpec(name)
c.Check(err, check.DeepEquals,
coordinate.ErrNoSuchWorkSpec{Name: name})
names, err = s.Namespace.WorkSpecNames()
c.Assert(err, check.IsNil)
c.Check(names, check.HasLen, 0)
dict = map[string]interface{}{
"name": name,
"min_gb": 0.1,
}
spec, err = s.Namespace.SetWorkSpec(dict)
c.Assert(err, check.IsNil)
c.Check(spec, check.NotNil)
c.Check(spec.Name(), check.Equals, name)
spec, err = s.Namespace.WorkSpec(name)
c.Assert(err, check.IsNil)
c.Check(spec, check.NotNil)
c.Check(spec.Name(), check.Equals, name)
data, err = spec.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, dict)
names, err = s.Namespace.WorkSpecNames()
c.Assert(err, check.IsNil)
c.Check(names, check.DeepEquals, []string{name})
err = s.Namespace.DestroyWorkSpec(name)
c.Check(err, check.IsNil)
spec, err = s.Namespace.WorkSpec(name)
c.Check(err, check.DeepEquals,
coordinate.ErrNoSuchWorkSpec{Name: name})
names, err = s.Namespace.WorkSpecNames()
c.Assert(err, check.IsNil)
c.Check(names, check.HasLen, 0)
err = s.Namespace.DestroyWorkSpec(name)
c.Check(err, check.DeepEquals,
coordinate.ErrNoSuchWorkSpec{Name: name})
}
// checkWorkUnitOrder verifies that getting all of the work possible
// retrieves work units in a specific order.
func checkWorkUnitOrder(
c *check.C,
worker coordinate.Worker,
spec coordinate.WorkSpec,
unitNames ...string,
) {
var processedUnits []string
for {
attempts, err := worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
if len(attempts) == 0 {
break
}
c.Assert(attempts, check.HasLen, 1)
attempt := attempts[0]
c.Check(attempt.WorkUnit().WorkSpec().Name(), check.Equals, spec.Name())
processedUnits = append(processedUnits, attempt.WorkUnit().Name())
err = attempt.Finish(nil)
c.Assert(err, check.IsNil)
}
c.Check(processedUnits, check.DeepEquals, unitNames)
}
// GetWorkSpecMeta returns a set of control options for a given work
// spec. The returned map has the full set of keys that
// ControlWorkSpec() will accept.
func (jobs *JobServer) GetWorkSpecMeta(workSpecName string) (result map[string]interface{}, _ string, err error) {
var (
workSpec coordinate.WorkSpec
meta coordinate.WorkSpecMeta
)
workSpec, err = jobs.Namespace.WorkSpec(workSpecName)
if err == nil {
meta, err = workSpec.Meta(false)
}
if err == nil {
result = make(map[string]interface{})
if meta.Paused {
result["status"] = Paused
} else {
result["status"] = Runnable
}
result["continuous"] = meta.Continuous
result["interval"] = meta.Interval.Seconds()
result["max_running"] = meta.MaxRunning
result["weight"] = meta.Weight
}
return
}
// TestChainingExpiry tests that, if an attempt finishes but is no
// longer the active attempt, then its successor work units will not
// be created.
func (s *Suite) TestChainingExpiry(c *check.C) {
var (
one, two coordinate.WorkSpec
err error
worker coordinate.Worker
unit coordinate.WorkUnit
attempts []coordinate.Attempt
)
one, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
c.Assert(err, check.IsNil)
two, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
"disabled": true,
})
c.Assert(err, check.IsNil)
worker, err = s.Namespace.Worker("worker")
c.Assert(err, check.IsNil)
// Create and perform a work unit, with no output
unit, err = one.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
attempt := attempts[0]
// But wait! We got preempted
err = unit.ClearActiveAttempt()
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
// Now, let the original attempt finish, trying to generate
// more outputs
err = attempt.Finish(map[string]interface{}{
"output": []string{"unit"},
})
c.Assert(err, check.IsNil)
// Since attempt is no longer active, this shouldn't generate
// new outputs
units, err := two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, check.HasLen, 0)
}
// TestChangeSpecData tests WorkSpec.SetData().
func (s *Suite) TestChangeSpecData(c *check.C) {
var (
err error
data map[string]interface{}
spec coordinate.WorkSpec
)
spec, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "spec",
"min_gb": 1,
})
c.Assert(err, check.IsNil)
c.Check(spec.Name(), check.Equals, "spec")
err = spec.SetData(map[string]interface{}{
"name": "spec",
"min_gb": 2,
"foo": "bar",
})
c.Assert(err, check.IsNil)
data, err = spec.Data()
c.Assert(err, check.IsNil)
c.Check(data["name"], check.Equals, "spec")
c.Check(data["min_gb"], Like, 2)
c.Check(data["foo"], check.Equals, "bar")
err = spec.SetData(map[string]interface{}{})
c.Assert(err, check.NotNil)
c.Check(err, check.Equals, coordinate.ErrNoWorkSpecName)
err = spec.SetData(map[string]interface{}{
"name": "name",
"min_gb": 3,
})
c.Assert(err, check.NotNil)
c.Check(err, check.Equals, coordinate.ErrChangedName)
data, err = spec.Data()
c.Assert(err, check.IsNil)
c.Check(data["name"], check.Equals, "spec")
c.Check(data["min_gb"], Like, 2)
c.Check(data["foo"], check.Equals, "bar")
}
// TestAttemptLifetime validates a basic attempt lifetime.
func (s *Suite) TestAttemptLifetime(c *check.C) {
var (
err error
data map[string]interface{}
attempt, attempt2 coordinate.Attempt
aStatus coordinate.AttemptStatus
spec coordinate.WorkSpec
unit coordinate.WorkUnit
worker coordinate.Worker
uStatus coordinate.WorkUnitStatus
)
spec, worker = s.makeWorkSpecAndWorker(c)
// Create a work unit
unit, err = spec.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
// The work unit should be "available"
uStatus, err = unit.Status()
c.Assert(err, check.IsNil)
c.Check(uStatus, check.Equals, coordinate.AvailableUnit)
// The work unit data should be defined but empty
data, err = unit.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.HasLen, 0)
// Get an attempt for it
attempts, err := worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
attempt = attempts[0]
// The work unit should be "pending"
uStatus, err = unit.Status()
c.Assert(err, check.IsNil)
c.Check(uStatus, check.Equals, coordinate.PendingUnit)
// The attempt should be "pending" too
aStatus, err = attempt.Status()
c.Assert(err, check.IsNil)
c.Check(aStatus, check.Equals, coordinate.Pending)
// The active attempt for the unit should match this
attempt2, err = unit.ActiveAttempt()
c.Assert(err, check.IsNil)
c.Check(attempt2, AttemptMatches, attempt)
// There should be one active attempt for the worker and it should
// also match
attempts, err = worker.ActiveAttempts()
c.Assert(err, check.IsNil)
c.Check(attempts, check.HasLen, 1)
if len(attempts) > 0 {
c.Check(attempts[0], AttemptMatches, attempt)
}
// The work unit data should (still) be defined but empty
data, err = unit.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.HasLen, 0)
// Now finish the attempt with some updated data
err = attempt.Finish(map[string]interface{}{
"outputs": []string{"yes"},
})
c.Assert(err, check.IsNil)
// The unit should report "finished"
uStatus, err = unit.Status()
c.Assert(err, check.IsNil)
c.Check(uStatus, check.Equals, coordinate.FinishedUnit)
// The attempt should report "finished"
aStatus, err = attempt.Status()
c.Assert(err, check.IsNil)
c.Check(aStatus, check.Equals, coordinate.Finished)
// The attempt should still be the active attempt for the unit
attempt2, err = unit.ActiveAttempt()
c.Assert(err, check.IsNil)
c.Check(attempt2, AttemptMatches, attempt)
// The attempt should not be in the active attempt list for the worker
attempts, err = worker.ActiveAttempts()
c.Assert(err, check.IsNil)
c.Check(attempts, check.HasLen, 0)
// Both the unit and the worker should have one archived attempt
attempts, err = unit.Attempts()
c.Assert(err, check.IsNil)
c.Check(attempts, check.HasLen, 1)
if len(attempts) > 0 {
c.Check(attempts[0], AttemptMatches, attempt)
}
attempts, err = worker.AllAttempts()
c.Assert(err, check.IsNil)
c.Check(attempts, check.HasLen, 1)
if len(attempts) > 0 {
c.Check(attempts[0], AttemptMatches, attempt)
}
// This should have updated the visible work unit data too
data, err = unit.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.HasLen, 1)
c.Check(data["outputs"], check.HasLen, 1)
c.Check(reflect.ValueOf(data["outputs"]).Index(0).Interface(),
check.Equals, "yes")
// For bonus points, force-clear the active attempt
err = unit.ClearActiveAttempt()
c.Assert(err, check.IsNil)
// This should have pushed the unit back to available
uStatus, err = unit.Status()
c.Assert(err, check.IsNil)
c.Check(uStatus, check.Equals, coordinate.AvailableUnit)
// This also should have reset the work unit data
data, err = unit.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.HasLen, 0)
// But, this should not have reset the historical attempts
attempts, err = unit.Attempts()
c.Assert(err, check.IsNil)
c.Check(attempts, check.HasLen, 1)
if len(attempts) > 0 {
c.Check(attempts[0], AttemptMatches, attempt)
}
attempts, err = worker.AllAttempts()
c.Assert(err, check.IsNil)
c.Check(attempts, check.HasLen, 1)
if len(attempts) > 0 {
c.Check(attempts[0], AttemptMatches, attempt)
}
}
// TestSetMeta tests the basic SetMeta() call and a couple of its
// documented oddities.
func (s *Suite) TestSetMeta(c *check.C) {
var (
err error
spec coordinate.WorkSpec
meta coordinate.WorkSpecMeta
)
spec, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "spec",
"min_gb": 1,
"continuous": true,
})
c.Assert(err, check.IsNil)
meta, err = spec.Meta(false)
c.Assert(err, check.IsNil)
c.Check(meta.Priority, check.Equals, 0)
c.Check(meta.Weight, check.Equals, 20)
c.Check(meta.Paused, check.Equals, false)
c.Check(meta.Continuous, check.Equals, true)
c.Check(meta.CanBeContinuous, check.Equals, true)
c.Check(meta.Interval, check.Equals, time.Duration(0))
c.Check(meta.NextContinuous, check.Equals, time.Time{})
c.Check(meta.MaxRunning, check.Equals, 0)
c.Check(meta.MaxAttemptsReturned, check.Equals, 0)
c.Check(meta.NextWorkSpecName, check.Equals, "")
c.Check(meta.AvailableCount, check.Equals, 0)
c.Check(meta.PendingCount, check.Equals, 0)
err = spec.SetMeta(coordinate.WorkSpecMeta{
Priority: 10,
Weight: 100,
Paused: true,
Continuous: false,
CanBeContinuous: false,
Interval: time.Duration(60) * time.Second,
MaxRunning: 10,
MaxAttemptsReturned: 1,
NextWorkSpecName: "then",
AvailableCount: 100,
PendingCount: 50,
})
c.Assert(err, check.IsNil)
meta, err = spec.Meta(false)
c.Assert(err, check.IsNil)
c.Check(meta.Priority, check.Equals, 10)
c.Check(meta.Weight, check.Equals, 100)
c.Check(meta.Paused, check.Equals, true)
c.Check(meta.Continuous, check.Equals, false)
// Cannot clear "can be continuous" flag
c.Check(meta.CanBeContinuous, check.Equals, true)
c.Check(meta.Interval, check.Equals, time.Duration(60)*time.Second)
c.Check(meta.NextContinuous, check.Equals, time.Time{})
c.Check(meta.MaxRunning, check.Equals, 10)
c.Check(meta.MaxAttemptsReturned, check.Equals, 1)
// Cannot change following work spec
c.Check(meta.NextWorkSpecName, check.Equals, "")
// Cannot set the counts
c.Check(meta.AvailableCount, check.Equals, 0)
c.Check(meta.PendingCount, check.Equals, 0)
}
// TestWorkUnitChaining tests that completing work units in one work spec
// will cause work units to appear in another, if so configured.
func (s *Suite) TestWorkUnitChaining(c *check.C) {
var (
err error
worker coordinate.Worker
one, two coordinate.WorkSpec
units map[string]coordinate.WorkUnit
attempts []coordinate.Attempt
data map[string]interface{}
ok bool
)
one, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
c.Assert(err, check.IsNil)
two, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
"disabled": true,
})
c.Assert(err, check.IsNil)
worker, err = s.Namespace.Worker("worker")
c.Assert(err, check.IsNil)
// Create and perform a work unit, with no output
_, err = one.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(nil)
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{})
// Create and perform a work unit, with a map output
_, err = one.AddWorkUnit("b", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(map[string]interface{}{
"output": map[string]interface{}{
"two_b": map[string]interface{}{"k": "v"},
},
})
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"two_b"})
if _, ok = units["two_b"]; ok {
data, err = units["two_b"].Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, map[string]interface{}{"k": "v"})
}
// Create and perform a work unit, with a slice output
_, err = one.AddWorkUnit("c", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(map[string]interface{}{
"output": []string{"two_c", "two_cc"},
})
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"two_b", "two_c", "two_cc"})
if _, ok = units["two_c"]; ok {
data, err = units["two_c"].Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, map[string]interface{}{})
}
// Put the output in the original work unit data
_, err = one.AddWorkUnit("d", map[string]interface{}{
"output": []string{"two_d"},
}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(nil)
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"two_b", "two_c", "two_cc", "two_d"})
}
// UpdateWorkUnit causes some state change in a work unit. If the
// work unit is pending, this is the principal interface to complete
// or renew it; if it is already complete this can cause it to be
// retried.
func (jobs *JobServer) UpdateWorkUnit(
workSpecName string,
workUnitKey string,
options map[string]interface{},
) (bool, string, error) {
// Note that in several corner cases, the behavior of this as
// written disagrees with Python coordinated's:
//
// * If neither "lease_time" nor "status" is specified,
// Python coordinated immediately returns False without
// checking if workUnitKey is valid
//
// * Python coordinated allows arbitrary status changes,
// including AVAILABLE -> FINISHED
//
// * This openly ignores "worker_id", as distinct from Python
// coordinated, which logs an obscure warning and changes it,
// but only on a renew
var (
attempt coordinate.Attempt
changed bool
err error
status coordinate.AttemptStatus
uwuOptions UpdateWorkUnitOptions
workSpec coordinate.WorkSpec
workUnit coordinate.WorkUnit
)
err = decode(&uwuOptions, options)
if err == nil {
workSpec, err = jobs.Namespace.WorkSpec(workSpecName)
}
if err == nil {
workUnit, err = workSpec.WorkUnit(workUnitKey)
}
if err == nil {
if workUnit == nil {
return false, fmt.Sprintf("no such work unit key=%v", workUnitKey), nil
}
}
if err == nil {
attempt, err = workUnit.ActiveAttempt()
}
if err == nil && attempt != nil {
status, err = attempt.Status()
}
if err == nil && attempt != nil {
if status == coordinate.Expired || status == coordinate.Retryable {
// The Python Coordinate API sees both of these
// statuses as "available", and we want to fall
// into the next block.
attempt = nil
}
}
if err == nil && attempt == nil {
// Caller is trying to manipulate an AVAILABLE work
// unit. Cowardly refuse to start a new attempt on
// their behalf, or to update the persistent work unit
// data this way. (In theory there's no reason we
// *couldn't* do either, though I'm not aware of any
// callers that do; add_work_unit will replace
// existing work units and is the more typical way to
// refresh data.)
err = errors.New("update_work_unit will not adjust an available work unit")
}
if err == nil {
switch status {
case coordinate.Pending:
changed = true // or there's an error
switch uwuOptions.Status {
case 0, Pending:
err = uwuRenew(attempt, uwuOptions)
case Available:
err = attempt.Expire(uwuOptions.Data)
case Finished:
err = attempt.Finish(uwuOptions.Data)
case Failed:
err = attempt.Fail(uwuOptions.Data)
default:
err = errors.New("update_work_unit invalid status")
}
case coordinate.Expired:
err = errors.New("update_work_unit logic error, trying to refresh expired unit")
case coordinate.Finished:
switch uwuOptions.Status {
case 0, Finished:
changed = false // no-op
case Available:
err = workUnit.ClearActiveAttempt()
changed = true
case Failed:
changed = false // see below
default:
err = errors.New("update_work_unit cannot change finished unit")
}
case coordinate.Failed:
switch uwuOptions.Status {
case 0, Failed:
changed = false // no-op
case Available: // "retry"
err = workUnit.ClearActiveAttempt()
changed = true
case Finished:
// The Python worker, with two separate
// processes, has a race wherein there
// could be 15 seconds to go, the parent
// kills off the child, and the child
// finishes successfully, all at the same
// time. In that case the successful
// finish should win.
err = attempt.Finish(nil)
changed = true
default:
err = errors.New("update_work_unit cannot change failed unit")
}
case coordinate.Retryable:
err = errors.New("update_work_unit logic error, trying to refresh retryable unit")
default:
err = fmt.Errorf("update_work_unit invalid attempt status %+v", status)
}
}
return changed && err == nil, "", err
}
// TestTwoWorkSpecsBasic ensures that two work specs can be created
// and have independent lifetimes.
func (s *Suite) TestTwoWorkSpecsBasic(c *check.C) {
var (
err error
dict1, dict2 map[string]interface{}
name1, name2 string
names []string
spec coordinate.WorkSpec
)
name1 = "spec1"
name2 = "spec2"
dict1 = map[string]interface{}{"name": name1, "min_gb": 1}
dict2 = map[string]interface{}{"name": name2, "min_gb": 2}
names, err = s.Namespace.WorkSpecNames()
c.Assert(err, check.IsNil)
c.Check(names, check.HasLen, 0)
spec, err = s.Namespace.SetWorkSpec(dict1)
c.Assert(err, check.IsNil)
names, err = s.Namespace.WorkSpecNames()
c.Assert(err, check.IsNil)
c.Check(names, check.DeepEquals, []string{name1})
spec, err = s.Namespace.SetWorkSpec(dict2)
c.Assert(err, check.IsNil)
names, err = s.Namespace.WorkSpecNames()
c.Assert(err, check.IsNil)
c.Check(names, check.HasLen, 2)
if len(names) > 0 {
if names[0] == name1 {
c.Check(names, check.DeepEquals, []string{name1, name2})
} else {
c.Check(names, check.DeepEquals, []string{name2, name1})
}
}
spec, err = s.Namespace.WorkSpec(name1)
c.Assert(err, check.IsNil)
c.Assert(spec, check.NotNil)
c.Check(spec.Name(), check.Equals, name1)
spec, err = s.Namespace.WorkSpec(name2)
c.Assert(err, check.IsNil)
c.Assert(spec, check.NotNil)
c.Check(spec.Name(), check.Equals, name2)
err = s.Namespace.DestroyWorkSpec(name1)
c.Assert(err, check.IsNil)
names, err = s.Namespace.WorkSpecNames()
c.Assert(err, check.IsNil)
c.Check(names, check.DeepEquals, []string{name2})
spec, err = s.Namespace.WorkSpec(name1)
c.Check(err, check.DeepEquals, coordinate.ErrNoSuchWorkSpec{Name: name1})
spec, err = s.Namespace.WorkSpec(name2)
c.Assert(err, check.IsNil)
c.Assert(spec, check.NotNil)
c.Check(spec.Name(), check.Equals, name2)
err = s.Namespace.DestroyWorkSpec(name2)
c.Assert(err, check.IsNil)
names, err = s.Namespace.WorkSpecNames()
c.Assert(err, check.IsNil)
c.Check(names, check.HasLen, 0)
spec, err = s.Namespace.WorkSpec(name1)
c.Check(err, check.DeepEquals, coordinate.ErrNoSuchWorkSpec{Name: name1})
spec, err = s.Namespace.WorkSpec(name2)
c.Check(err, check.DeepEquals, coordinate.ErrNoSuchWorkSpec{Name: name2})
}
// TestChainingMixed uses a combination of strings and tuples in its
// "output" data.
func (s *Suite) TestChainingMixed(c *check.C) {
var (
one, two coordinate.WorkSpec
worker coordinate.Worker
attempts []coordinate.Attempt
units map[string]coordinate.WorkUnit
unit coordinate.WorkUnit
data map[string]interface{}
priority float64
ok bool
err error
)
one, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
c.Assert(err, check.IsNil)
two, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
})
c.Assert(err, check.IsNil)
worker, err = s.Namespace.Worker("worker")
c.Assert(err, check.IsNil)
_, err = one.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(map[string]interface{}{
"output": []interface{}{
"key",
cborrpc.PythonTuple{Items: []interface{}{
"key",
map[string]interface{}{
"data": "x",
},
map[string]interface{}{
"priority": 10.0,
},
}},
},
})
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"key"})
if unit, ok = units["key"]; ok {
data, err = unit.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, map[string]interface{}{"data": "x"})
priority, err = unit.Priority()
c.Assert(err, check.IsNil)
c.Check(priority, check.Equals, 10.0)
}
}
// ------------------------------------------------------------------------
// Coordinate setup helpers:
func createWorkUnits(spec coordinate.WorkSpec, n int, c *check.C) {
for i := 0; i < n; i++ {
_, err := spec.AddWorkUnit(fmt.Sprintf("u%v", i), map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
}
}
// TestChainingTwoStep separately renews an attempt to insert an output
// key, then finishes the work unit; it should still chain.
func (s *Suite) TestChainingTwoStep(c *check.C) {
var (
one, two coordinate.WorkSpec
worker coordinate.Worker
attempts []coordinate.Attempt
units map[string]coordinate.WorkUnit
unit coordinate.WorkUnit
data map[string]interface{}
priority float64
ok bool
err error
)
one, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
c.Assert(err, check.IsNil)
two, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
})
c.Assert(err, check.IsNil)
worker, err = s.Namespace.Worker("worker")
c.Assert(err, check.IsNil)
_, err = one.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Renew(time.Duration(900)*time.Second,
map[string]interface{}{
"output": []interface{}{
[]byte{1, 2, 3, 4},
cborrpc.PythonTuple{Items: []interface{}{
[]byte{1, 2, 3, 4},
map[interface{}]interface{}{},
map[interface{}]interface{}{
"priority": 0,
},
}},
},
})
c.Assert(err, check.IsNil)
err = attempts[0].Finish(nil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"\x01\x02\x03\x04"})
if unit, ok = units["\x01\x02\x03\x04"]; ok {
data, err = unit.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, map[string]interface{}{})
priority, err = unit.Priority()
c.Assert(err, check.IsNil)
c.Check(priority, check.Equals, 0.0)
}
}
// TestTrivialWorkUnitFlow tests work unit creation, deletion, and existence.
func (s *Suite) TestTrivialWorkUnitFlow(c *check.C) {
var (
count int
err error
spec coordinate.WorkSpec
unit coordinate.WorkUnit
units map[string]coordinate.WorkUnit
)
spec, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "spec",
"min_gb": 1,
})
c.Assert(err, check.IsNil)
unit, err = spec.AddWorkUnit("unit", map[string]interface{}{}, 0)
c.Assert(err, check.IsNil)
c.Check(unit.Name(), check.Equals, "unit")
c.Check(unit.WorkSpec().Name(), check.Equals, "spec")
unit, err = spec.WorkUnit("unit")
c.Assert(err, check.IsNil)
c.Check(unit.Name(), check.Equals, "unit")
c.Check(unit.WorkSpec().Name(), check.Equals, "spec")
units, err = spec.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, check.HasLen, 1)
c.Check(units["unit"], check.NotNil)
c.Check(units["unit"].Name(), check.Equals, "unit")
c.Check(units["unit"].WorkSpec().Name(), check.Equals, "spec")
count, err = spec.DeleteWorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(count, check.Equals, 1)
unit, err = spec.WorkUnit("unit")
c.Assert(err, check.IsNil)
c.Check(unit, check.IsNil)
units, err = spec.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, check.HasLen, 0)
}