// TestChainingMixed uses a combination of strings and tuples in its
// "output" data.
func TestChainingMixed(t *testing.T) {
var (
one, two coordinate.WorkSpec
attempt coordinate.Attempt
units map[string]coordinate.WorkUnit
err error
)
sts := SimpleTestSetup{
NamespaceName: "TestChainingMixed",
WorkerName: "worker",
}
sts.SetUp(t)
defer sts.TearDown(t)
one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
if !assert.NoError(t, err) {
return
}
two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
})
if !assert.NoError(t, err) {
return
}
_, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{})
assert.NoError(t, err)
sts.WorkSpec = one
attempt = sts.RequestOneAttempt(t)
err = attempt.Finish(map[string]interface{}{
"output": []interface{}{
"key",
cborrpc.PythonTuple{Items: []interface{}{
"key",
map[string]interface{}{
"data": "x",
},
map[string]interface{}{
"priority": 10.0,
},
}},
},
})
assert.NoError(t, err)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
if assert.NoError(t, err) {
if assert.Contains(t, units, "key") {
DataMatches(t, units["key"], map[string]interface{}{"data": "x"})
UnitHasPriority(t, units["key"], 10.0)
}
}
}
// TestChainingExpiry tests that, if an attempt finishes but is no
// longer the active attempt, then its successor work units will not
// be created.
func TestChainingExpiry(t *testing.T) {
var (
one, two coordinate.WorkSpec
err error
unit coordinate.WorkUnit
)
sts := SimpleTestSetup{
NamespaceName: "TestChainingExpiry",
WorkerName: "worker",
}
sts.SetUp(t)
defer sts.TearDown(t)
one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
if !assert.NoError(t, err) {
return
}
sts.WorkSpec = one
two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
"disabled": true,
})
if !assert.NoError(t, err) {
return
}
// Create and perform a work unit, with no output
unit, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{})
if !assert.NoError(t, err) {
return
}
attempt := sts.RequestOneAttempt(t)
// But wait! We got preempted
err = unit.ClearActiveAttempt()
assert.NoError(t, err)
sts.RequestOneAttempt(t)
// Now, let the original attempt finish, trying to generate
// more outputs
err = attempt.Finish(map[string]interface{}{
"output": []string{"unit"},
})
assert.NoError(t, err)
// Since attempt is no longer active, this shouldn't generate
// new outputs
units, err := two.WorkUnits(coordinate.WorkUnitQuery{})
if assert.NoError(t, err) {
assert.Empty(t, units)
}
}
// TestChainingTwoStep separately renews an attempt to insert an output
// key, then finishes the work unit; it should still chain.
func TestChainingTwoStep(t *testing.T) {
var (
one, two coordinate.WorkSpec
attempt coordinate.Attempt
units map[string]coordinate.WorkUnit
unit coordinate.WorkUnit
err error
)
sts := SimpleTestSetup{
NamespaceName: "TestChainingTwoStep",
WorkerName: "worker",
}
sts.SetUp(t)
defer sts.TearDown(t)
one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
if !assert.NoError(t, err) {
return
}
two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
})
if !assert.NoError(t, err) {
return
}
_, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{})
assert.NoError(t, err)
sts.WorkSpec = one
attempt = sts.RequestOneAttempt(t)
err = attempt.Renew(900*time.Second,
map[string]interface{}{
"output": []interface{}{
[]byte{1, 2, 3, 4},
cborrpc.PythonTuple{Items: []interface{}{
[]byte{1, 2, 3, 4},
map[interface{}]interface{}{},
map[interface{}]interface{}{
"priority": 0,
},
}},
},
})
assert.NoError(t, err)
err = attempt.Finish(nil)
assert.NoError(t, err)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
if assert.NoError(t, err) {
if assert.Contains(t, units, "\x01\x02\x03\x04") {
unit = units["\x01\x02\x03\x04"]
DataEmpty(t, unit)
UnitHasPriority(t, unit, 0.0)
}
}
}
// TestWorkUnitChaining tests that completing work units in one work spec
// will cause work units to appear in another, if so configured.
func TestWorkUnitChaining(t *testing.T) {
var (
err error
one, two coordinate.WorkSpec
units map[string]coordinate.WorkUnit
attempt coordinate.Attempt
)
sts := SimpleTestSetup{
NamespaceName: "TestWorkUnitChaining",
WorkerName: "worker",
}
sts.SetUp(t)
defer sts.TearDown(t)
one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
"name": "one",
"then": "two",
})
if !assert.NoError(t, err) {
return
}
// RequestAttempts always returns this
sts.WorkSpec = one
two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
"name": "two",
"disabled": true,
})
if !assert.NoError(t, err) {
return
}
// Create and perform a work unit, with no output
_, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{})
assert.NoError(t, err)
sts.WorkSpec = one
attempt = sts.RequestOneAttempt(t)
err = attempt.Finish(nil)
assert.NoError(t, err)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
if assert.NoError(t, err) {
assert.Empty(t, units)
}
// Create and perform a work unit, with a map output
_, err = one.AddWorkUnit("b", map[string]interface{}{}, coordinate.WorkUnitMeta{})
assert.NoError(t, err)
attempt = sts.RequestOneAttempt(t)
err = attempt.Finish(map[string]interface{}{
"output": map[string]interface{}{
"two_b": map[string]interface{}{"k": "v"},
},
})
assert.NoError(t, err)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
if assert.NoError(t, err) {
assert.Len(t, units, 1)
if assert.Contains(t, units, "two_b") {
DataMatches(t, units["two_b"], map[string]interface{}{"k": "v"})
}
}
// Create and perform a work unit, with a slice output
_, err = one.AddWorkUnit("c", map[string]interface{}{}, coordinate.WorkUnitMeta{})
assert.NoError(t, err)
attempt = sts.RequestOneAttempt(t)
err = attempt.Finish(map[string]interface{}{
"output": []string{"two_c", "two_cc"},
})
assert.NoError(t, err)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
if assert.NoError(t, err) {
assert.Len(t, units, 3)
assert.Contains(t, units, "two_b")
assert.Contains(t, units, "two_cc")
if assert.Contains(t, units, "two_c") {
DataEmpty(t, units["two_c"])
}
}
// Put the output in the original work unit data
_, err = one.AddWorkUnit("d", map[string]interface{}{
"output": []string{"two_d"},
}, coordinate.WorkUnitMeta{})
assert.NoError(t, err)
attempt = sts.RequestOneAttempt(t)
err = attempt.Finish(nil)
assert.NoError(t, err)
units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
if assert.NoError(t, err) {
assert.Len(t, units, 4)
assert.Contains(t, units, "two_b")
assert.Contains(t, units, "two_c")
assert.Contains(t, units, "two_cc")
assert.Contains(t, units, "two_d")
}
}
