func (s *ProvisionerSuite) TestProvisionerStopRetryingIfDying(c *gc.C) { // Create the error injection channel and inject // a retryable error errorInjectionChannel := make(chan error, 1) p := s.newEnvironProvisioner(c) // Don't refer the stop. We will manually stop and verify the result. // patch the dummy provider error injection channel cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) defer cleanup() retryableError := instance.NewRetryableCreationError("container failed to start and was destroyed") errorInjectionChannel <- retryableError m, err := s.addMachine() c.Assert(err, jc.ErrorIsNil) time.Sleep(coretesting.ShortWait) stop(c, p) statusInfo, err := m.Status() c.Assert(err, jc.ErrorIsNil) c.Assert(statusInfo.Status, gc.Equals, state.StatusPending) s.checkNoOperations(c) }
func (s *ProvisionerSuite) TestProvisionerSucceedStartInstanceWithInjectedWrappedRetryableCreationError(c *gc.C) { // Set the retry delay to 0, and retry count to 1 to keep tests short s.PatchValue(provisioner.RetryStrategyDelay, 0*time.Second) s.PatchValue(provisioner.RetryStrategyCount, 1) // create the error injection channel errorInjectionChannel := make(chan error, 1) c.Assert(errorInjectionChannel, gc.NotNil) p := s.newEnvironProvisioner(c) defer stop(c, p) // patch the dummy provider error injection channel cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) defer cleanup() // send the error message once // - instance creation should succeed retryableError := errors.Wrap(errors.New(""), instance.NewRetryableCreationError("container failed to start and was destroyed")) errorInjectionChannel <- retryableError m, err := s.addMachine() c.Assert(err, jc.ErrorIsNil) s.checkStartInstanceNoSecureConnection(c, m) }
func (s *ProvisionerSuite) TestProvisionerFailedStartInstanceWithInjectedCreationError(c *gc.C) { // Set the retry delay to 0, and retry count to 2 to keep tests short s.PatchValue(provisioner.RetryStrategyDelay, 0*time.Second) s.PatchValue(provisioner.RetryStrategyCount, 2) // create the error injection channel errorInjectionChannel := make(chan error, 3) p := s.newEnvironProvisioner(c) defer stop(c, p) // patch the dummy provider error injection channel cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) defer cleanup() retryableError := instance.NewRetryableCreationError("container failed to start and was destroyed") destroyError := errors.New("container failed to start and failed to destroy: manual cleanup of containers needed") // send the error message three times, because the provisioner will retry twice as patched above. errorInjectionChannel <- retryableError errorInjectionChannel <- retryableError errorInjectionChannel <- destroyError m, err := s.addMachine() c.Assert(err, jc.ErrorIsNil) s.checkNoOperations(c) t0 := time.Now() for time.Since(t0) < coretesting.LongWait { // And check the machine status is set to error. statusInfo, err := m.Status() c.Assert(err, jc.ErrorIsNil) if statusInfo.Status == state.StatusPending { time.Sleep(coretesting.ShortWait) continue } c.Assert(statusInfo.Status, gc.Equals, state.StatusError) // check that the status matches the error message c.Assert(statusInfo.Message, gc.Equals, destroyError.Error()) return } c.Fatal("Test took too long to complete") }
func (s *ProvisionerSuite) TestProvisionerSucceedStartInstanceWithInjectedRetryableCreationError(c *gc.C) { // create the error injection channel errorInjectionChannel := make(chan error, 1) c.Assert(errorInjectionChannel, gc.NotNil) p := s.newEnvironProvisioner(c) defer stop(c, p) // patch the dummy provider error injection channel cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) defer cleanup() // send the error message once // - instance creation should succeed retryableError := instance.NewRetryableCreationError("container failed to start and was destroyed") errorInjectionChannel <- retryableError m, err := s.addMachine() c.Assert(err, jc.ErrorIsNil) s.checkStartInstanceNoSecureConnection(c, m) }
func (s *ProvisionerSuite) TestProvisionerFailedStartInstanceWithInjectedCreationError(c *gc.C) { // create the error injection channel errorInjectionChannel := make(chan error, 2) p := s.newEnvironProvisioner(c) defer stop(c, p) // patch the dummy provider error injection channel cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) defer cleanup() retryableError := instance.NewRetryableCreationError("container failed to start and was destroyed") destroyError := errors.New("container failed to start and failed to destroy: manual cleanup of containers needed") // send the error message TWICE, because the provisioner will retry only ONCE errorInjectionChannel <- retryableError errorInjectionChannel <- destroyError m, err := s.addMachine() c.Assert(err, jc.ErrorIsNil) s.checkNoOperations(c) t0 := time.Now() for time.Since(t0) < coretesting.LongWait { // And check the machine status is set to error. statusInfo, err := m.Status() c.Assert(err, jc.ErrorIsNil) if statusInfo.Status == state.StatusPending { time.Sleep(coretesting.ShortWait) continue } c.Assert(statusInfo.Status, gc.Equals, state.StatusError) // check that the status matches the error message c.Assert(statusInfo.Message, gc.Equals, destroyError.Error()) break } }
func (s *ProvisionerSuite) TestProvisionerFailStartInstanceWithInjectedNonRetryableCreationError(c *gc.C) { // create the error injection channel errorInjectionChannel := make(chan error, 1) c.Assert(errorInjectionChannel, gc.NotNil) p := s.newEnvironProvisioner(c) defer stop(c, p) // patch the dummy provider error injection channel cleanup := dummy.PatchTransientErrorInjectionChannel(errorInjectionChannel) defer cleanup() // send the error message once nonRetryableError := errors.New("some nonretryable error") errorInjectionChannel <- nonRetryableError m, err := s.addMachine() c.Assert(err, jc.ErrorIsNil) s.checkNoOperations(c) t0 := time.Now() for time.Since(t0) < coretesting.LongWait { // And check the machine status is set to error. statusInfo, err := m.Status() c.Assert(err, jc.ErrorIsNil) if statusInfo.Status == state.StatusPending { time.Sleep(coretesting.ShortWait) continue } c.Assert(statusInfo.Status, gc.Equals, state.StatusError) // check that the status matches the error message c.Assert(statusInfo.Message, gc.Equals, nonRetryableError.Error()) return } c.Fatal("Test took too long to complete") }