// TestChainingExpiry tests that, if an attempt finishes but is no
// longer the active attempt, then its successor work units will not
// be created.
func (s *Suite) TestChainingExpiry(c *check.C) {
var (
one, two coordinate.WorkSpec
err error
worker coordinate.Worker
unit coordinate.WorkUnit
attempts []coordinate.Attempt
)
one, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
c.Assert(err, check.IsNil)
two, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
"disabled": true,
})
c.Assert(err, check.IsNil)
worker, err = s.Namespace.Worker("worker")
c.Assert(err, check.IsNil)
// Create and perform a work unit, with no output
unit, err = one.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
attempt := attempts[0]
// But wait! We got preempted
err = unit.ClearActiveAttempt()
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
// Now, let the original attempt finish, trying to generate
// more outputs
err = attempt.Finish(map[string]interface{}{
"output": []string{"unit"},
})
c.Assert(err, check.IsNil)
// Since attempt is no longer active, this shouldn't generate
// new outputs
units, err := two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, check.HasLen, 0)
}
// TestTrivialWorkUnitFlow tests work unit creation, deletion, and existence.
func (s *Suite) TestTrivialWorkUnitFlow(c *check.C) {
var (
count int
err error
spec coordinate.WorkSpec
unit coordinate.WorkUnit
units map[string]coordinate.WorkUnit
)
spec, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "spec",
"min_gb": 1,
})
c.Assert(err, check.IsNil)
unit, err = spec.AddWorkUnit("unit", map[string]interface{}{}, 0)
c.Assert(err, check.IsNil)
c.Check(unit.Name(), check.Equals, "unit")
c.Check(unit.WorkSpec().Name(), check.Equals, "spec")
unit, err = spec.WorkUnit("unit")
c.Assert(err, check.IsNil)
c.Check(unit.Name(), check.Equals, "unit")
c.Check(unit.WorkSpec().Name(), check.Equals, "spec")
units, err = spec.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, check.HasLen, 1)
c.Check(units["unit"], check.NotNil)
c.Check(units["unit"].Name(), check.Equals, "unit")
c.Check(units["unit"].WorkSpec().Name(), check.Equals, "spec")
count, err = spec.DeleteWorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(count, check.Equals, 1)
unit, err = spec.WorkUnit("unit")
c.Assert(err, check.IsNil)
c.Check(unit, check.IsNil)
units, err = spec.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, check.HasLen, 0)
}
// TestChainingMixed uses a combination of strings and tuples in its
// "output" data.
func (s *Suite) TestChainingMixed(c *check.C) {
var (
one, two coordinate.WorkSpec
worker coordinate.Worker
attempts []coordinate.Attempt
units map[string]coordinate.WorkUnit
unit coordinate.WorkUnit
data map[string]interface{}
priority float64
ok bool
err error
)
one, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
c.Assert(err, check.IsNil)
two, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
})
c.Assert(err, check.IsNil)
worker, err = s.Namespace.Worker("worker")
c.Assert(err, check.IsNil)
_, err = one.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(map[string]interface{}{
"output": []interface{}{
"key",
cborrpc.PythonTuple{Items: []interface{}{
"key",
map[string]interface{}{
"data": "x",
},
map[string]interface{}{
"priority": 10.0,
},
}},
},
})
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"key"})
if unit, ok = units["key"]; ok {
data, err = unit.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, map[string]interface{}{"data": "x"})
priority, err = unit.Priority()
c.Assert(err, check.IsNil)
c.Check(priority, check.Equals, 10.0)
}
}
// TestChainingTwoStep separately renews an attempt to insert an output
// key, then finishes the work unit; it should still chain.
func (s *Suite) TestChainingTwoStep(c *check.C) {
var (
one, two coordinate.WorkSpec
worker coordinate.Worker
attempts []coordinate.Attempt
units map[string]coordinate.WorkUnit
unit coordinate.WorkUnit
data map[string]interface{}
priority float64
ok bool
err error
)
one, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
c.Assert(err, check.IsNil)
two, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
})
c.Assert(err, check.IsNil)
worker, err = s.Namespace.Worker("worker")
c.Assert(err, check.IsNil)
_, err = one.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Renew(time.Duration(900)*time.Second,
map[string]interface{}{
"output": []interface{}{
[]byte{1, 2, 3, 4},
cborrpc.PythonTuple{Items: []interface{}{
[]byte{1, 2, 3, 4},
map[interface{}]interface{}{},
map[interface{}]interface{}{
"priority": 0,
},
}},
},
})
c.Assert(err, check.IsNil)
err = attempts[0].Finish(nil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"\x01\x02\x03\x04"})
if unit, ok = units["\x01\x02\x03\x04"]; ok {
data, err = unit.Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, map[string]interface{}{})
priority, err = unit.Priority()
c.Assert(err, check.IsNil)
c.Check(priority, check.Equals, 0.0)
}
}
// TestWorkUnitChaining tests that completing work units in one work spec
// will cause work units to appear in another, if so configured.
func (s *Suite) TestWorkUnitChaining(c *check.C) {
var (
err error
worker coordinate.Worker
one, two coordinate.WorkSpec
units map[string]coordinate.WorkUnit
attempts []coordinate.Attempt
data map[string]interface{}
ok bool
)
one, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
c.Assert(err, check.IsNil)
two, err = s.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
"disabled": true,
})
c.Assert(err, check.IsNil)
worker, err = s.Namespace.Worker("worker")
c.Assert(err, check.IsNil)
// Create and perform a work unit, with no output
_, err = one.AddWorkUnit("a", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(nil)
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{})
// Create and perform a work unit, with a map output
_, err = one.AddWorkUnit("b", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(map[string]interface{}{
"output": map[string]interface{}{
"two_b": map[string]interface{}{"k": "v"},
},
})
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"two_b"})
if _, ok = units["two_b"]; ok {
data, err = units["two_b"].Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, map[string]interface{}{"k": "v"})
}
// Create and perform a work unit, with a slice output
_, err = one.AddWorkUnit("c", map[string]interface{}{}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(map[string]interface{}{
"output": []string{"two_c", "two_cc"},
})
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"two_b", "two_c", "two_cc"})
if _, ok = units["two_c"]; ok {
data, err = units["two_c"].Data()
c.Assert(err, check.IsNil)
c.Check(data, check.DeepEquals, map[string]interface{}{})
}
// Put the output in the original work unit data
_, err = one.AddWorkUnit("d", map[string]interface{}{
"output": []string{"two_d"},
}, 0.0)
c.Assert(err, check.IsNil)
attempts, err = worker.RequestAttempts(coordinate.AttemptRequest{})
c.Assert(err, check.IsNil)
c.Assert(attempts, check.HasLen, 1)
err = attempts[0].Finish(nil)
c.Assert(err, check.IsNil)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
c.Assert(err, check.IsNil)
c.Check(units, HasKeys, []string{"two_b", "two_c", "two_cc", "two_d"})
}