コード例 #1
0
ファイル: work.go プロジェクト: diffeo/go-coordinate
// UpdateWorkUnit causes some state change in a work unit.  If the
// work unit is pending, this is the principal interface to complete
// or renew it; if it is already complete this can cause it to be
// retried.
func (jobs *JobServer) UpdateWorkUnit(
	workSpecName string,
	workUnitKey string,
	options map[string]interface{},
) (bool, string, error) {
	// Note that in several corner cases, the behavior of this as
	// written disagrees with Python coordinated's:
	//
	// * If neither "lease_time" nor "status" is specified,
	//   Python coordinated immediately returns False without
	//   checking if workUnitKey is valid
	//
	// * Python coordinated allows arbitrary status changes,
	//   including AVAILABLE -> FINISHED
	//
	// * This openly ignores "worker_id", as distinct from Python
	//   coordinated, which logs an obscure warning and changes it,
	//   but only on a renew
	var (
		attempt    coordinate.Attempt
		changed    bool
		err        error
		status     coordinate.AttemptStatus
		uwuOptions UpdateWorkUnitOptions
		workSpec   coordinate.WorkSpec
		workUnit   coordinate.WorkUnit
	)
	err = decode(&uwuOptions, options)
	if err == nil {
		workSpec, err = jobs.Namespace.WorkSpec(workSpecName)
	}
	if err == nil {
		workUnit, err = workSpec.WorkUnit(workUnitKey)
	}
	if err == nil {
		if workUnit == nil {
			return false, fmt.Sprintf("no such work unit key=%v", workUnitKey), nil
		}
	}
	if err == nil {
		attempt, err = workUnit.ActiveAttempt()
	}
	if err == nil && attempt != nil {
		status, err = attempt.Status()
	}
	if err == nil && attempt != nil {
		if status == coordinate.Expired || status == coordinate.Retryable {
			// The Python Coordinate API sees both of these
			// statuses as "available", and we want to fall
			// into the next block.
			attempt = nil
		}
	}
	if err == nil && attempt == nil {
		// Caller is trying to manipulate an AVAILABLE work
		// unit.  Either they are trying to change the work
		// unit data in place, or they are trying to jump a
		// work unit directly to a completed state.  (The
		// latter is possible during the Python work unit
		// parent cleanup, if the timing is bad.)
		if uwuOptions.Status == Available || uwuOptions.Status == 0 {
			// The only thing we are doing is changing the
			// work unit data.
			if uwuOptions.Data != nil {
				meta, err := workUnit.Meta()
				if err == nil {
					_, err = workSpec.AddWorkUnit(workUnit.Name(), uwuOptions.Data, meta)
				}
				if err == nil {
					changed = true
				}
			}
			return changed && err == nil, "", err
		}
		// Otherwise we are trying to transition to another
		// state; so force-create an attempt.
		worker, err := jobs.Namespace.Worker(uwuOptions.WorkerID)
		if err == nil {
			attempt, err = worker.MakeAttempt(workUnit, uwuOptions.LeaseDuration())
			status = coordinate.Pending
		}
	}
	if err == nil {
		switch status {
		case coordinate.Pending:
			changed = true // or there's an error
			switch uwuOptions.Status {
			case 0, Pending:
				err = attempt.Renew(uwuOptions.LeaseDuration(), uwuOptions.Data)
			case Available:
				err = attempt.Expire(uwuOptions.Data)
			case Finished:
				err = attempt.Finish(uwuOptions.Data)
			case Failed:
				err = attempt.Fail(uwuOptions.Data)
			default:
				err = errors.New("update_work_unit invalid status")
			}
		case coordinate.Expired:
			err = errors.New("update_work_unit logic error, trying to refresh expired unit")
		case coordinate.Finished:
			switch uwuOptions.Status {
			case 0, Finished:
				changed = false // no-op
			case Available:
				err = workUnit.ClearActiveAttempt()
				changed = true
			case Failed:
				changed = false // see below
			default:
				err = errors.New("update_work_unit cannot change finished unit")
			}
		case coordinate.Failed:
			switch uwuOptions.Status {
			case 0, Failed:
				changed = false // no-op
			case Available: // "retry"
				err = workUnit.ClearActiveAttempt()
				changed = true
			case Finished:
				// The Python worker, with two separate
				// processes, has a race wherein there
				// could be 15 seconds to go, the parent
				// kills off the child, and the child
				// finishes successfully, all at the same
				// time.  In that case the successful
				// finish should win.
				err = attempt.Finish(nil)
				changed = true
			default:
				err = errors.New("update_work_unit cannot change failed unit")
			}
		case coordinate.Retryable:
			err = errors.New("update_work_unit logic error, trying to refresh retryable unit")
		default:
			err = fmt.Errorf("update_work_unit invalid attempt status %+v", status)
		}
	}
	return changed && err == nil, "", err
}
コード例 #2
0
ファイル: attempt.go プロジェクト: diffeo/go-coordinate
// TestChainingTwoStep separately renews an attempt to insert an output
// key, then finishes the work unit; it should still chain.
func TestChainingTwoStep(t *testing.T) {
	var (
		one, two coordinate.WorkSpec
		attempt  coordinate.Attempt
		units    map[string]coordinate.WorkUnit
		unit     coordinate.WorkUnit
		err      error
	)

	sts := SimpleTestSetup{
		NamespaceName: "TestChainingTwoStep",
		WorkerName:    "worker",
	}
	sts.SetUp(t)
	defer sts.TearDown(t)

	one, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
		"name": "one",
		"then": "two",
	})
	if !assert.NoError(t, err) {
		return
	}

	two, err = sts.Namespace.SetWorkSpec(map[string]interface{}{
		"name": "two",
	})
	if !assert.NoError(t, err) {
		return
	}

	_, err = one.AddWorkUnit("a", map[string]interface{}{}, coordinate.WorkUnitMeta{})
	assert.NoError(t, err)

	sts.WorkSpec = one
	attempt = sts.RequestOneAttempt(t)
	err = attempt.Renew(900*time.Second,
		map[string]interface{}{
			"output": []interface{}{
				[]byte{1, 2, 3, 4},
				cborrpc.PythonTuple{Items: []interface{}{
					[]byte{1, 2, 3, 4},
					map[interface{}]interface{}{},
					map[interface{}]interface{}{
						"priority": 0,
					},
				}},
			},
		})
	assert.NoError(t, err)

	err = attempt.Finish(nil)
	assert.NoError(t, err)

	units, err = two.WorkUnits(coordinate.WorkUnitQuery{})
	if assert.NoError(t, err) {
		if assert.Contains(t, units, "\x01\x02\x03\x04") {
			unit = units["\x01\x02\x03\x04"]
			DataEmpty(t, unit)
			UnitHasPriority(t, unit, 0.0)
		}
	}
}