func (w *worker) SetParent(parent coordinate.Worker) error {
parentName := ""
if parent != nil {
parentName = parent.Name()
}
repr := restdata.Worker{Parent: &parentName}
return w.Put(repr, nil)
}
// wrapWorker returns a cache.worker object for a specific upstream
// Worker.
func (ns *namespace) wrapWorker(upstream coordinate.Worker) *worker {
// This cannot fail: it can only fail if the embedded function
// returns an error, and the embedded function never fails
downstream, _ := ns.workers.Get(upstream.Name(), func(string) (named, error) {
return newWorker(upstream, ns), nil
})
return downstream.(*worker)
}
// GetWork requests one or more work units to perform. The work unit
// attempts are associated with workerID, which need not have been
// previously registered. If there is no work to do, may return
// neither work nor an error.
//
// Each work unit is returned as a cborrpc.PythonTuple holding the
// work spec name, work unit key as a byte slice, and work unit data
// dictionary. If options does not contain "max_jobs" or if that
// value is 1, returns a tuple or nil, otherwise returns a slice of
// tuples (maybe 1 or none).
func (jobs *JobServer) GetWork(workerID string, options map[string]interface{}) (interface{}, string, error) {
// This is the Big Kahuna. The Python Coordinate server tries
// to be extra clever with its return value, returning None if
// there is no work, a concrete value if one work unit was
// requested, and a list if more than one was requested, and
// this same rule is enforced in the client code. So, this will
// return either exactly one PythonTuple or a list of PythonTuple.
var (
attempts []coordinate.Attempt
err error
gwOptions GetWorkOptions
worker coordinate.Worker
)
err = decode(&gwOptions, options)
if err == nil {
worker, err = jobs.Namespace.Worker(workerID)
}
if err == nil {
if gwOptions.MaxJobs < 1 {
gwOptions.MaxJobs = 1
}
req := coordinate.AttemptRequest{
NumberOfWorkUnits: gwOptions.MaxJobs,
Runtimes: []string{""},
WorkSpecs: gwOptions.WorkSpecNames,
}
attempts, err = worker.RequestAttempts(req)
}
if err != nil {
return nil, "", err
}
// successful return
if gwOptions.MaxJobs == 1 {
if len(attempts) == 0 {
tuple := cborrpc.PythonTuple{
Items: []interface{}{nil, nil, nil},
}
return tuple, "", nil
}
if len(attempts) == 1 {
tuple, err := getWorkTuple(attempts[0])
if err != nil {
return nil, "", err
}
return tuple, "", nil
}
}
result := make([]cborrpc.PythonTuple, len(attempts))
for i, attempt := range attempts {
tuple, err := getWorkTuple(attempt)
if err != nil {
return nil, "", err
}
result[i] = tuple
}
return result, "", nil
}
func (api *restAPI) fillWorkerShort(namespace coordinate.Namespace, worker coordinate.Worker, short *restdata.WorkerShort) error {
short.Name = worker.Name()
return buildURLs(api.Router,
"namespace", namespace.Name(),
"worker", short.Name,
).
URL(&short.URL, "worker").
Error
}
func (api *restAPI) WorkerPut(ctx *context, in interface{}) (interface{}, error) {
var err error
repr, valid := in.(restdata.Worker)
if !valid {
return nil, errUnmarshal
}
// Did the parent change?
if repr.Parent != nil {
var oldParent, newParent coordinate.Worker
oldParent, err = ctx.Worker.Parent()
oldParentName := ""
if err == nil && oldParent != nil {
oldParentName = oldParent.Name()
}
if err == nil && *repr.Parent != oldParentName {
if *repr.Parent != "" {
newParent, err = ctx.Namespace.Worker(*repr.Parent)
}
if err == nil {
err = ctx.Worker.SetParent(newParent)
}
}
}
// Do we need to deactivate ourselves? Or update?
var wasActive bool
if err == nil {
wasActive, err = ctx.Worker.Active()
}
if err == nil && repr.Active {
// May as well update; checking everything else is
// a little irritating
err = ctx.Worker.Update(repr.Data, repr.LastUpdate, repr.Expiration, repr.Mode)
} else if err == nil && wasActive {
// was active, not active now (else we hit the previous block)
err = ctx.Worker.Deactivate()
}
return nil, err
}
// doWork gets attempts and runs them. It assumes it is running in its
// own goroutine. It signals gotWork when the call to RequestAttempts
// returns, and signals finished immediately before returning.
func (w *Worker) doWork(id string, worker coordinate.Worker, ctx context.Context, gotWork chan<- bool, finished chan<- string) {
// When we finish, signal the finished channel with our own ID
defer func() {
finished <- id
}()
attempts, err := worker.RequestAttempts(coordinate.AttemptRequest{
Runtimes: []string{"go"},
NumberOfWorkUnits: w.MaxAttempts,
})
if err != nil {
// Handle the error if we can, but otherwise act just like
// we got no attempts back
if w.ErrorHandler != nil {
w.ErrorHandler(err)
}
gotWork <- false
return
}
if len(attempts) == 0 {
// Nothing to do
gotWork <- false
return
}
// Otherwise we have actual work (and at least one attempt).
gotWork <- true
// See if we can find a task for the work spec
spec := attempts[0].WorkUnit().WorkSpec()
task := spec.Name()
data, err := spec.Data()
if err == nil {
aTask, present := data["task"]
if present {
bTask, ok := aTask.(string)
if ok {
task = bTask
}
}
}
// Try to find the task function
var taskFn func(context.Context, []coordinate.Attempt)
if err == nil {
taskFn = w.Tasks[task]
if taskFn == nil {
err = fmt.Errorf("No such task function %q", task)
}
}
if err == nil {
taskCtx, cancellation := context.WithCancel(ctx)
w.cancellations[id] = cancellation
taskFn(taskCtx, attempts)
// It appears to be recommended to call this; calling
// it multiple times is documented to have no effect
cancellation()
} else {
failure := map[string]interface{}{
"traceback": err.Error(),
}
// Try to fail all the attempts, ignoring errors
for _, attempt := range attempts {
_ = attempt.Fail(failure)
}
}
}
// TestWorkerAdoption hands a child worker to a new parent.
func TestWorkerAdoption(t *testing.T) {
var (
err error
child, oldParent, newParent, worker coordinate.Worker
kids []coordinate.Worker
)
sts := SimpleTestSetup{NamespaceName: "TestWorkerAdoption"}
sts.SetUp(t)
defer sts.TearDown(t)
// Create the worker objects
child, err = sts.Namespace.Worker("child")
if !assert.NoError(t, err) {
return
}
oldParent, err = sts.Namespace.Worker("old")
if !assert.NoError(t, err) {
return
}
newParent, err = sts.Namespace.Worker("new")
if !assert.NoError(t, err) {
return
}
// Set up the original ancestry
err = child.SetParent(oldParent)
assert.NoError(t, err)
// Move it to the new parent
err = child.SetParent(newParent)
assert.NoError(t, err)
// Checks
worker, err = child.Parent()
if assert.NoError(t, err) && assert.NotNil(t, worker) {
assert.Equal(t, "new", worker.Name())
}
kids, err = child.Children()
if assert.NoError(t, err) {
assert.Empty(t, kids)
}
worker, err = oldParent.Parent()
if assert.NoError(t, err) {
assert.Nil(t, worker)
}
kids, err = oldParent.Children()
if assert.NoError(t, err) {
assert.Empty(t, kids)
}
worker, err = newParent.Parent()
if assert.NoError(t, err) {
assert.Nil(t, worker)
}
kids, err = newParent.Children()
if assert.NoError(t, err) && assert.Len(t, kids, 1) {
assert.Equal(t, "child", kids[0].Name())
}
}
// TestWorkerAncestry does basic tests on worker parents and children.
func TestWorkerAncestry(t *testing.T) {
var (
err error
parent, child, worker coordinate.Worker
kids []coordinate.Worker
)
sts := SimpleTestSetup{NamespaceName: "TestWorkerAncestry"}
sts.SetUp(t)
defer sts.TearDown(t)
// start in the middle
parent, err = sts.Namespace.Worker("parent")
if !assert.NoError(t, err) {
return
}
worker, err = parent.Parent()
if assert.NoError(t, err) {
assert.Nil(t, worker)
}
kids, err = parent.Children()
if assert.NoError(t, err) {
assert.Empty(t, kids)
}
// Create a child
child, err = sts.Namespace.Worker("child")
if !assert.NoError(t, err) {
return
}
err = child.SetParent(parent)
assert.NoError(t, err)
// this should update the parent metadata
worker, err = parent.Parent()
if assert.NoError(t, err) {
assert.Nil(t, worker)
}
kids, err = parent.Children()
if assert.NoError(t, err) && assert.Len(t, kids, 1) {
assert.Equal(t, "child", kids[0].Name())
}
// and also the child metadata
worker, err = child.Parent()
if assert.NoError(t, err) && assert.NotNil(t, worker) {
assert.Equal(t, "parent", worker.Name())
}
kids, err = child.Children()
if assert.NoError(t, err) {
assert.Empty(t, kids)
}
}
func (api *restAPI) fillWorker(namespace coordinate.Namespace, worker coordinate.Worker, result *restdata.Worker) error {
err := api.fillWorkerShort(namespace, worker, &result.WorkerShort)
if err == nil {
err = buildURLs(api.Router,
"namespace", namespace.Name(),
"worker", worker.Name(),
).
URL(&result.RequestAttemptsURL, "workerRequestAttempts").
URL(&result.MakeAttemptURL, "workerMakeAttempt").
URL(&result.ActiveAttemptsURL, "workerActiveAttempts").
URL(&result.AllAttemptsURL, "workerAllAttempts").
URL(&result.ChildAttemptsURL, "workerChildAttempts").
Error
}
var parent coordinate.Worker
if err == nil {
parent, err = worker.Parent()
}
if err == nil && parent != nil {
parentName := parent.Name()
result.Parent = &parentName
err = buildURLs(api.Router,
"namespace", namespace.Name(),
"worker", parent.Name(),
).
URL(&result.ParentURL, "worker").
Error
}
var children []coordinate.Worker
if err == nil {
children, err = worker.Children()
}
if err == nil {
result.ChildURLs = make([]string, len(children))
for i, child := range children {
err = buildURLs(api.Router,
"namespace", namespace.Name(),
"worker", child.Name(),
).
URL(&result.ChildURLs[i], "worker").
Error
if err != nil {
break
}
}
}
if err == nil {
result.Active, err = worker.Active()
}
if err == nil {
result.Mode, err = worker.Mode()
}
if err == nil {
result.Data, err = worker.Data()
}
if err == nil {
result.Expiration, err = worker.Expiration()
}
if err == nil {
result.LastUpdate, err = worker.LastUpdate()
}
return err
}