// TestChainingMixed uses a combination of strings and tuples in its // "output" data. func TestChainingMixed(t *testing.T) { var ( one, two coordinate.WorkSpec attempt coordinate.Attempt units map[string]coordinate.WorkUnit err error ) sts := SimpleTestSetup{ NamespaceName: "TestChainingMixed", WorkerName: "worker", } sts.SetUp(t) defer sts.TearDown(t) one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{ "name": "one", "then": "two", }) if !assert.NoError(t, err) { return } two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{ "name": "two", }) if !assert.NoError(t, err) { return } _, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{}) assert.NoError(t, err) sts.WorkSpec = one attempt = sts.RequestOneAttempt(t) err = attempt.Finish(map[string]interface{}{ "output": []interface{}{ "key", cborrpc.PythonTuple{Items: []interface{}{ "key", map[string]interface{}{ "data": "x", }, map[string]interface{}{ "priority": 10.0, }, }}, }, }) assert.NoError(t, err) units, err = two.WorkUnits(coordinate.WorkUnitQuery{}) if assert.NoError(t, err) { if assert.Contains(t, units, "key") { DataMatches(t, units["key"], map[string]interface{}{"data": "x"}) UnitHasPriority(t, units["key"], 10.0) } } }
// TestChainingExpiry tests that, if an attempt finishes but is no // longer the active attempt, then its successor work units will not // be created. func TestChainingExpiry(t *testing.T) { var ( one, two coordinate.WorkSpec err error unit coordinate.WorkUnit ) sts := SimpleTestSetup{ NamespaceName: "TestChainingExpiry", WorkerName: "worker", } sts.SetUp(t) defer sts.TearDown(t) one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{ "name": "one", "then": "two", }) if !assert.NoError(t, err) { return } sts.WorkSpec = one two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{ "name": "two", "disabled": true, }) if !assert.NoError(t, err) { return } // Create and perform a work unit, with no output unit, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{}) if !assert.NoError(t, err) { return } attempt := sts.RequestOneAttempt(t) // But wait! We got preempted err = unit.ClearActiveAttempt() assert.NoError(t, err) sts.RequestOneAttempt(t) // Now, let the original attempt finish, trying to generate // more outputs err = attempt.Finish(map[string]interface{}{ "output": []string{"unit"}, }) assert.NoError(t, err) // Since attempt is no longer active, this shouldn't generate // new outputs units, err := two.WorkUnits(coordinate.WorkUnitQuery{}) if assert.NoError(t, err) { assert.Empty(t, units) } }
// TestChainingTwoStep separately renews an attempt to insert an output // key, then finishes the work unit; it should still chain. func TestChainingTwoStep(t *testing.T) { var ( one, two coordinate.WorkSpec attempt coordinate.Attempt units map[string]coordinate.WorkUnit unit coordinate.WorkUnit err error ) sts := SimpleTestSetup{ NamespaceName: "TestChainingTwoStep", WorkerName: "worker", } sts.SetUp(t) defer sts.TearDown(t) one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{ "name": "one", "then": "two", }) if !assert.NoError(t, err) { return } two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{ "name": "two", }) if !assert.NoError(t, err) { return } _, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{}) assert.NoError(t, err) sts.WorkSpec = one attempt = sts.RequestOneAttempt(t) err = attempt.Renew(900*time.Second, map[string]interface{}{ "output": []interface{}{ []byte{1, 2, 3, 4}, cborrpc.PythonTuple{Items: []interface{}{ []byte{1, 2, 3, 4}, map[interface{}]interface{}{}, map[interface{}]interface{}{ "priority": 0, }, }}, }, }) assert.NoError(t, err) err = attempt.Finish(nil) assert.NoError(t, err) units, err = two.WorkUnits(coordinate.WorkUnitQuery{}) if assert.NoError(t, err) { if assert.Contains(t, units, "\x01\x02\x03\x04") { unit = units["\x01\x02\x03\x04"] DataEmpty(t, unit) UnitHasPriority(t, unit, 0.0) } } }
// TestWorkUnitChaining tests that completing work units in one work spec // will cause work units to appear in another, if so configured. func TestWorkUnitChaining(t *testing.T) { var ( err error one, two coordinate.WorkSpec units map[string]coordinate.WorkUnit attempt coordinate.Attempt ) sts := SimpleTestSetup{ NamespaceName: "TestWorkUnitChaining", WorkerName: "worker", } sts.SetUp(t) defer sts.TearDown(t) one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{ "name": "one", "then": "two", }) if !assert.NoError(t, err) { return } // RequestAttempts always returns this sts.WorkSpec = one two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{ "name": "two", "disabled": true, }) if !assert.NoError(t, err) { return } // Create and perform a work unit, with no output _, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{}) assert.NoError(t, err) sts.WorkSpec = one attempt = sts.RequestOneAttempt(t) err = attempt.Finish(nil) assert.NoError(t, err) units, err = two.WorkUnits(coordinate.WorkUnitQuery{}) if assert.NoError(t, err) { assert.Empty(t, units) } // Create and perform a work unit, with a map output _, err = one.AddWorkUnit("b", map[string]interface{}{}, coordinate.WorkUnitMeta{}) assert.NoError(t, err) attempt = sts.RequestOneAttempt(t) err = attempt.Finish(map[string]interface{}{ "output": map[string]interface{}{ "two_b": map[string]interface{}{"k": "v"}, }, }) assert.NoError(t, err) units, err = two.WorkUnits(coordinate.WorkUnitQuery{}) if assert.NoError(t, err) { assert.Len(t, units, 1) if assert.Contains(t, units, "two_b") { DataMatches(t, units["two_b"], map[string]interface{}{"k": "v"}) } } // Create and perform a work unit, with a slice output _, err = one.AddWorkUnit("c", map[string]interface{}{}, coordinate.WorkUnitMeta{}) assert.NoError(t, err) attempt = sts.RequestOneAttempt(t) err = attempt.Finish(map[string]interface{}{ "output": []string{"two_c", "two_cc"}, }) assert.NoError(t, err) units, err = two.WorkUnits(coordinate.WorkUnitQuery{}) if assert.NoError(t, err) { assert.Len(t, units, 3) assert.Contains(t, units, "two_b") assert.Contains(t, units, "two_cc") if assert.Contains(t, units, "two_c") { DataEmpty(t, units["two_c"]) } } // Put the output in the original work unit data _, err = one.AddWorkUnit("d", map[string]interface{}{ "output": []string{"two_d"}, }, coordinate.WorkUnitMeta{}) assert.NoError(t, err) attempt = sts.RequestOneAttempt(t) err = attempt.Finish(nil) assert.NoError(t, err) units, err = two.WorkUnits(coordinate.WorkUnitQuery{}) if assert.NoError(t, err) { assert.Len(t, units, 4) assert.Contains(t, units, "two_b") assert.Contains(t, units, "two_c") assert.Contains(t, units, "two_cc") assert.Contains(t, units, "two_d") } }