// NextOp is defined on the Resolver interface.
func (s *storageResolver) NextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
var changed []names.StorageTag
for tag, storage := range remoteState.Storage {
life, ok := s.life[tag]
if !ok || life != storage.Life {
s.life[tag] = storage.Life
changed = append(changed, tag)
}
}
for tag := range s.life {
if _, ok := remoteState.Storage[tag]; !ok {
changed = append(changed, tag)
delete(s.life, tag)
}
}
if len(changed) > 0 {
return opFactory.NewUpdateStorage(changed)
}
if !localState.Installed && s.storage.Pending() == 0 {
logger.Infof("initial storage attachments ready")
}
return s.nextOp(localState, remoteState, opFactory)
}
func (s *uniterResolver) nextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
switch remoteState.Life {
case params.Alive:
case params.Dying:
// Normally we handle relations last, but if we're dying we
// must ensure that all relations are broken first.
op, err := s.config.Relations.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
// We're not in a hook error and the unit is Dying,
// so we should proceed to tear down.
//
// TODO(axw) move logic for cascading destruction of
// subordinates, relation units and storage
// attachments into state, via cleanups.
if localState.Started {
return opFactory.NewRunHook(hook.Info{Kind: hooks.Stop})
}
fallthrough
case params.Dead:
// The unit is dying/dead and stopped, so tell the uniter
// to terminate.
return nil, resolver.ErrTerminate
}
// Now that storage hooks have run at least once, before anything else,
// we need to run the install hook.
// TODO(cmars): remove !localState.Started. It's here as a temporary
// measure because unit agent upgrades aren't being performed yet.
if !localState.Installed && !localState.Started {
return opFactory.NewRunHook(hook.Info{Kind: hooks.Install})
}
if charmModified(localState, remoteState) {
return opFactory.NewUpgrade(remoteState.CharmURL)
}
if localState.ConfigVersion != remoteState.ConfigVersion {
return opFactory.NewRunHook(hook.Info{Kind: hooks.ConfigChanged})
}
op, err := s.config.Relations.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
// UpdateStatus hook runs if nothing else needs to.
if localState.UpdateStatusVersion != remoteState.UpdateStatusVersion {
return opFactory.NewRunHook(hook.Info{Kind: hooks.UpdateStatus})
}
return nil, resolver.ErrNoOperation
}
// NextOp implements resolver.Resolver.
func (s *relationsResolver) NextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
hook, err := s.relations.NextHook(localState, remoteState)
if err != nil {
return nil, errors.Trace(err)
}
return opFactory.NewRunHook(hook)
}
func (s *storageResolver) nextHookOp(
tag names.StorageTag,
snap remotestate.StorageSnapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
logger.Debugf("next hook op for %v: %+v", tag, snap)
if !snap.Attached {
return nil, resolver.ErrNoOperation
}
storageAttachment, ok := s.storage.storageAttachments[tag]
if !ok {
return nil, resolver.ErrNoOperation
}
switch snap.Life {
case params.Alive:
if storageAttachment.attached {
// Storage attachments currently do not change
// (apart from lifecycle) after being provisioned.
// We don't process unprovisioned storage here,
// so there's nothing to do.
return nil, resolver.ErrNoOperation
}
case params.Dying:
if !storageAttachment.attached {
// Nothing to do: attachment is dying, but
// the storage-attached hook has not been
// consumed.
return nil, resolver.ErrNoOperation
}
case params.Dead:
// Storage must have been Dying to become Dead;
// no further action is required.
return nil, resolver.ErrNoOperation
}
hookInfo := hook.Info{
StorageId: tag.Id(),
}
if snap.Life == params.Alive {
hookInfo.Kind = hooks.StorageAttached
} else {
hookInfo.Kind = hooks.StorageDetaching
}
context := &contextStorage{
tag: tag,
kind: storage.StorageKind(snap.Kind),
location: snap.Location,
}
storageAttachment.ContextStorageAttachment = context
s.storage.storageAttachments[tag] = storageAttachment
logger.Debugf("queued hook: %v", hookInfo)
return opFactory.NewRunHook(hookInfo)
}
// NextOp implements the resolver.Resolver interface.
func (r *actionsResolver) NextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
nextAction, err := nextAction(remoteState.Actions, localState.CompletedActions)
if err != nil {
return nil, err
}
switch localState.Kind {
case operation.RunHook:
// We can still run actions if the unit is in a hook error state.
if localState.Step == operation.Pending {
return opFactory.NewAction(nextAction)
}
case operation.RunAction:
if localState.Hook != nil {
logger.Infof("found incomplete action %q; ignoring", localState.ActionId)
logger.Infof("recommitting prior %q hook", localState.Hook.Kind)
return opFactory.NewSkipHook(*localState.Hook)
} else {
logger.Infof("%q hook is nil", operation.RunAction)
return opFactory.NewFailAction(*localState.ActionId)
}
case operation.Continue:
return opFactory.NewAction(nextAction)
}
return nil, resolver.ErrNoOperation
}
// nextOpConflicted is called after an upgrade operation has failed, and hasn't
// yet been resolved or reverted. When in this mode, the resolver will only
// consider those two possibilities for progressing.
func (s *uniterResolver) nextOpConflicted(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
if remoteState.ResolvedMode != params.ResolvedNone {
if err := s.config.ClearResolved(); err != nil {
return nil, errors.Trace(err)
}
return opFactory.NewResolvedUpgrade(localState.CharmURL)
}
if remoteState.ForceCharmUpgrade && charmModified(localState, remoteState) {
return opFactory.NewRevertUpgrade(remoteState.CharmURL)
}
return nil, resolver.ErrWaiting
}
func (s *uniterResolver) nextOpHookError(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
// Report the hook error.
if err := s.config.ReportHookError(*localState.Hook); err != nil {
return nil, errors.Trace(err)
}
if remoteState.ForceCharmUpgrade && *localState.CharmURL != *remoteState.CharmURL {
logger.Debugf("upgrade from %v to %v", localState.CharmURL, remoteState.CharmURL)
return opFactory.NewUpgrade(remoteState.CharmURL)
}
switch remoteState.ResolvedMode {
case params.ResolvedNone:
if remoteState.RetryHookVersion > localState.RetryHookVersion {
// We've been asked to retry: clear the hook timer
// started state so we'll restart it if this fails.
//
// If the hook fails again, we'll re-enter this method
// with the retry hook versions equal and restart the
// timer. If the hook succeeds, we'll enter nextOp
// and stop the timer.
s.retryHookTimerStarted = false
return opFactory.NewRunHook(*localState.Hook)
}
if !s.retryHookTimerStarted {
// We haven't yet started a retry timer, so start one
// now. If we retry and fail, retryHookTimerStarted is
// cleared so that we'll still start it again.
s.config.StartRetryHookTimer()
s.retryHookTimerStarted = true
}
return nil, resolver.ErrNoOperation
case params.ResolvedRetryHooks:
s.config.StopRetryHookTimer()
s.retryHookTimerStarted = false
if err := s.config.ClearResolved(); err != nil {
return nil, errors.Trace(err)
}
return opFactory.NewRunHook(*localState.Hook)
case params.ResolvedNoHooks:
s.config.StopRetryHookTimer()
s.retryHookTimerStarted = false
if err := s.config.ClearResolved(); err != nil {
return nil, errors.Trace(err)
}
return opFactory.NewSkipHook(*localState.Hook)
default:
return nil, errors.Errorf(
"unknown resolved mode %q", remoteState.ResolvedMode,
)
}
}
// nextOpConflicted is called after an upgrade operation has failed, and hasn't
// yet been resolved or reverted. When in this mode, the resolver will only
// consider those two possibilities for progressing.
func (s *uniterResolver) nextOpConflicted(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
if remoteState.ResolvedMode != params.ResolvedNone {
if err := s.clearResolved(); err != nil {
return nil, errors.Trace(err)
}
return opFactory.NewResolvedUpgrade(localState.CharmURL)
}
if remoteState.ForceCharmUpgrade && *localState.CharmURL != *remoteState.CharmURL {
logger.Debugf("upgrade from %v to %v", localState.CharmURL, remoteState.CharmURL)
return opFactory.NewRevertUpgrade(remoteState.CharmURL)
}
return nil, resolver.ErrWaiting
}
// NextOp is part of the resolver.Resolver interface.
func (s *commandsResolver) NextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
if len(remoteState.Commands) == 0 {
return nil, resolver.ErrNoOperation
}
id := remoteState.Commands[0]
op, err := opFactory.NewCommands(s.commands.GetCommand(id))
if err != nil {
return nil, err
}
commandCompleted := func() {
s.commands.RemoveCommand(id)
s.commandCompleted(id)
}
return &commandCompleter{op, commandCompleted}, nil
}
// NextOp implements the resolver.Resolver interface.
func (r *actionsResolver) NextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
nextAction, err := nextAction(remoteState.Actions, localState.CompletedActions)
if err != nil {
return nil, err
}
switch localState.Kind {
case operation.RunHook:
// We can still run actions if the unit is in a hook error state.
if localState.Step == operation.Pending {
return opFactory.NewAction(nextAction)
}
case operation.RunAction:
// TODO(fwereade): we *should* handle interrupted actions, and make sure
// they're marked as failed, but that's not for now.
if localState.Hook != nil {
logger.Infof("found incomplete action %q; ignoring", localState.ActionId)
logger.Infof("recommitting prior %q hook", localState.Hook.Kind)
return opFactory.NewSkipHook(*localState.Hook)
} else {
logger.Infof("%q hook is nil", operation.RunAction)
}
case operation.Continue:
return opFactory.NewAction(nextAction)
}
return nil, resolver.ErrNoOperation
}
func (s *uniterResolver) nextOpHookError(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
// Report the hook error.
if err := s.reportHookError(*localState.Hook); err != nil {
return nil, errors.Trace(err)
}
if remoteState.ForceCharmUpgrade && *localState.CharmURL != *remoteState.CharmURL {
logger.Debugf("upgrade from %v to %v", localState.CharmURL, remoteState.CharmURL)
return opFactory.NewUpgrade(remoteState.CharmURL)
}
switch remoteState.ResolvedMode {
case params.ResolvedNone:
return nil, resolver.ErrNoOperation
case params.ResolvedRetryHooks:
if err := s.clearResolved(); err != nil {
return nil, errors.Trace(err)
}
return opFactory.NewRunHook(*localState.Hook)
case params.ResolvedNoHooks:
if err := s.clearResolved(); err != nil {
return nil, errors.Trace(err)
}
return opFactory.NewSkipHook(*localState.Hook)
default:
return nil, errors.Errorf(
"unknown resolved mode %q", remoteState.ResolvedMode,
)
}
}
// NextOp is defined on the Resolver interface.
func (l *leadershipResolver) NextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
// TODO(wallyworld) - maybe this can occur before install
if !localState.Installed {
return nil, resolver.ErrNoOperation
}
// Check for any leadership change, and enact it if possible.
logger.Infof("checking leadership status")
// If we've already accepted leadership, we don't need to do it again.
canAcceptLeader := !localState.Leader
if remoteState.Life == params.Dying {
canAcceptLeader = false
} else {
// If we're in an unexpected mode (eg pending hook) we shouldn't try either.
if localState.Kind != operation.Continue {
canAcceptLeader = false
}
}
switch {
case remoteState.Leader && canAcceptLeader:
return opFactory.NewAcceptLeadership()
// If we're the leader but should not be any longer, or
// if the unit is dying, we should resign leadership.
case localState.Leader && (!remoteState.Leader || remoteState.Life == params.Dying):
return opFactory.NewResignLeadership()
}
if localState.Kind == operation.Continue {
// We want to run the leader settings hook if we're
// not the leader and the settings have changed.
if !localState.Leader && localState.LeaderSettingsVersion != remoteState.LeaderSettingsVersion {
return opFactory.NewRunHook(hook.Info{Kind: hook.LeaderSettingsChanged})
}
}
logger.Infof("leadership status is up-to-date")
return nil, resolver.ErrNoOperation
}
func (s *uniterResolver) NextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
if remoteState.Life == params.Dead || localState.Stopped {
return nil, resolver.ErrTerminate
}
if localState.Kind == operation.Upgrade {
if localState.Conflicted {
return s.nextOpConflicted(localState, remoteState, opFactory)
}
logger.Infof("resuming charm upgrade")
return opFactory.NewUpgrade(localState.CharmURL)
}
if localState.Restart {
// We've just run the upgrade op, which will change the
// unit's charm URL. We need to restart the resolver
// loop so that we start watching the correct events.
return nil, resolver.ErrRestart
}
if s.retryHookTimerStarted && (localState.Kind != operation.RunHook || localState.Step != operation.Pending) {
// The hook-retry timer is running, but there is no pending
// hook operation. We're not in an error state, so stop the
// timer now to reset the backoff state.
s.config.StopRetryHookTimer()
s.retryHookTimerStarted = false
}
op, err := s.config.Leadership.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
op, err = s.config.Actions.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
op, err = s.config.Commands.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
op, err = s.config.Storage.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
switch localState.Kind {
case operation.RunHook:
switch localState.Step {
case operation.Pending:
logger.Infof("awaiting error resolution for %q hook", localState.Hook.Kind)
return s.nextOpHookError(localState, remoteState, opFactory)
case operation.Queued:
logger.Infof("found queued %q hook", localState.Hook.Kind)
if localState.Hook.Kind == hooks.Install {
// Special case: handle install in nextOp,
// so we do nothing when the unit is dying.
return s.nextOp(localState, remoteState, opFactory)
}
return opFactory.NewRunHook(*localState.Hook)
case operation.Done:
logger.Infof("committing %q hook", localState.Hook.Kind)
return opFactory.NewSkipHook(*localState.Hook)
default:
return nil, errors.Errorf("unknown operation step %v", localState.Step)
}
case operation.Continue:
logger.Infof("no operations in progress; waiting for changes")
return s.nextOp(localState, remoteState, opFactory)
default:
return nil, errors.Errorf("unknown operation kind %v", localState.Kind)
}
}
func (s *storageResolver) nextHookOp(
tag names.StorageTag,
snap remotestate.StorageSnapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
logger.Debugf("next hook op for %v: %+v", tag, snap)
if snap.Life == params.Dead {
// Storage must have been Dying to become Dead;
// no further action is required.
return nil, resolver.ErrNoOperation
}
hookInfo := hook.Info{StorageId: tag.Id()}
switch snap.Life {
case params.Alive:
storageAttachment, ok := s.storage.storageAttachments[tag]
if ok && storageAttachment.attached {
// Once the storage is attached, we only care about
// lifecycle state changes.
return nil, resolver.ErrNoOperation
}
// The storage-attached hook has not been committed, so add the
// storage to the pending set.
s.storage.pending.Add(tag)
if !snap.Attached {
// The storage attachment has not been provisioned yet,
// so just ignore it for now. We'll be notified again
// when it has been provisioned.
return nil, resolver.ErrNoOperation
}
// The storage is alive, but we haven't previously run the
// "storage-attached" hook. Do so now.
hookInfo.Kind = hooks.StorageAttached
case params.Dying:
storageAttachment, ok := s.storage.storageAttachments[tag]
if !ok || !storageAttachment.attached {
// Nothing to do: attachment is dying, but
// the storage-attached hook has not been
// issued.
return nil, resolver.ErrNoOperation
}
// The storage is dying, but we haven't previously run the
// "storage-detached" hook. Do so now.
hookInfo.Kind = hooks.StorageDetaching
}
// Update the local state to reflect what we're about to report
// to a hook.
stateFile, err := readStateFile(s.storage.storageStateDir, tag)
if err != nil {
return nil, errors.Trace(err)
}
s.storage.storageAttachments[tag] = storageAttachment{
stateFile, &contextStorage{
tag: tag,
kind: storage.StorageKind(snap.Kind),
location: snap.Location,
},
}
return opFactory.NewRunHook(hookInfo)
}
func (s *uniterResolver) NextOp(
localState resolver.LocalState,
remoteState remotestate.Snapshot,
opFactory operation.Factory,
) (operation.Operation, error) {
if remoteState.Life == params.Dead || localState.Stopped {
return nil, resolver.ErrTerminate
}
if localState.Kind == operation.Upgrade {
if localState.Conflicted {
return s.nextOpConflicted(localState, remoteState, opFactory)
}
logger.Infof("resuming charm upgrade")
return opFactory.NewUpgrade(localState.CharmURL)
}
if localState.Restart {
// We've just run the upgrade op, which will change the
// unit's charm URL. We need to restart the resolver
// loop so that we start watching the correct events.
return nil, resolver.ErrRestart
}
if localState.Kind == operation.Continue {
if err := s.fixDeployer(); err != nil {
return nil, errors.Trace(err)
}
}
op, err := s.leadershipResolver.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
op, err = s.actionsResolver.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
op, err = s.storageResolver.NextOp(localState, remoteState, opFactory)
if errors.Cause(err) != resolver.ErrNoOperation {
return op, err
}
switch localState.Kind {
case operation.RunHook:
switch localState.Step {
case operation.Pending:
logger.Infof("awaiting error resolution for %q hook", localState.Hook.Kind)
return s.nextOpHookError(localState, remoteState, opFactory)
case operation.Queued:
logger.Infof("found queued %q hook", localState.Hook.Kind)
if localState.Hook.Kind == hooks.Install {
// Special case: handle install in nextOp,
// so we do nothing when the unit is dying.
return s.nextOp(localState, remoteState, opFactory)
}
return opFactory.NewRunHook(*localState.Hook)
case operation.Done:
logger.Infof("committing %q hook", localState.Hook.Kind)
return opFactory.NewSkipHook(*localState.Hook)
default:
return nil, errors.Errorf("unknown operation step %v", localState.Step)
}
case operation.Continue:
logger.Infof("no operations in progress; waiting for changes")
return s.nextOp(localState, remoteState, opFactory)
default:
return nil, errors.Errorf("unknown operation kind %v", localState.Kind)
}
}