func (s *storageProvisionerSuite) TestAttachVolumeRetry(c *gc.C) {
volumeInfoSet := make(chan interface{})
volumeAccessor := newMockVolumeAccessor()
volumeAccessor.provisionedMachines["machine-1"] = instance.Id("already-provisioned-1")
volumeAccessor.setVolumeInfo = func(volumes []params.Volume) ([]params.ErrorResult, error) {
defer close(volumeInfoSet)
return make([]params.ErrorResult, len(volumes)), nil
}
volumeAttachmentInfoSet := make(chan interface{})
volumeAccessor.setVolumeAttachmentInfo = func(volumeAttachments []params.VolumeAttachment) ([]params.ErrorResult, error) {
defer close(volumeAttachmentInfoSet)
return make([]params.ErrorResult, len(volumeAttachments)), nil
}
// mockFunc's After will progress the current time by the specified
// duration and signal the channel immediately.
clock := &mockClock{}
var attachVolumeTimes []time.Time
s.provider.attachVolumesFunc = func(args []storage.VolumeAttachmentParams) ([]storage.AttachVolumesResult, error) {
attachVolumeTimes = append(attachVolumeTimes, clock.Now())
if len(attachVolumeTimes) < 10 {
return []storage.AttachVolumesResult{{Error: errors.New("badness")}}, nil
}
return []storage.AttachVolumesResult{{
VolumeAttachment: &storage.VolumeAttachment{
args[0].Volume,
args[0].Machine,
storage.VolumeAttachmentInfo{
DeviceName: "/dev/sda1",
},
},
}}, nil
}
args := &workerArgs{volumes: volumeAccessor, clock: clock}
worker := newStorageProvisioner(c, args)
defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
defer worker.Kill()
volumeAccessor.attachmentsWatcher.changes <- []params.MachineStorageId{{
MachineTag: "machine-1", AttachmentTag: "volume-1",
}}
volumeAccessor.volumesWatcher.changes <- []string{"1"}
args.environ.watcher.changes <- struct{}{}
waitChannel(c, volumeInfoSet, "waiting for volume info to be set")
waitChannel(c, volumeAttachmentInfoSet, "waiting for volume attachments to be set")
c.Assert(attachVolumeTimes, gc.HasLen, 10)
// The first attempt should have been immediate: T0.
c.Assert(attachVolumeTimes[0], gc.Equals, time.Time{})
delays := make([]time.Duration, len(attachVolumeTimes)-1)
for i := range attachVolumeTimes[1:] {
delays[i] = attachVolumeTimes[i+1].Sub(attachVolumeTimes[i])
}
c.Assert(delays, jc.DeepEquals, []time.Duration{
30 * time.Second,
1 * time.Minute,
2 * time.Minute,
4 * time.Minute,
8 * time.Minute,
16 * time.Minute,
30 * time.Minute, // ceiling reached
30 * time.Minute,
30 * time.Minute,
})
c.Assert(args.statusSetter.args, jc.DeepEquals, []params.EntityStatusArgs{
{Tag: "volume-1", Status: "attaching", Info: ""}, // CreateVolumes
{Tag: "volume-1", Status: "attaching", Info: "badness"}, // AttachVolumes
{Tag: "volume-1", Status: "attaching", Info: "badness"},
{Tag: "volume-1", Status: "attaching", Info: "badness"},
{Tag: "volume-1", Status: "attaching", Info: "badness"},
{Tag: "volume-1", Status: "attaching", Info: "badness"},
{Tag: "volume-1", Status: "attaching", Info: "badness"},
{Tag: "volume-1", Status: "attaching", Info: "badness"},
{Tag: "volume-1", Status: "attaching", Info: "badness"},
{Tag: "volume-1", Status: "attaching", Info: "badness"},
{Tag: "volume-1", Status: "attached", Info: ""},
})
}
func (s *storageProvisionerSuite) TestDetachVolumesRetry(c *gc.C) {
machine := names.NewMachineTag("1")
volume := names.NewVolumeTag("1")
attachmentId := params.MachineStorageId{
MachineTag: machine.String(), AttachmentTag: volume.String(),
}
volumeAccessor := newMockVolumeAccessor()
volumeAccessor.provisionedAttachments[attachmentId] = params.VolumeAttachment{
MachineTag: machine.String(),
VolumeTag: volume.String(),
}
volumeAccessor.provisionedVolumes[volume.String()] = params.Volume{
VolumeTag: volume.String(),
Info: params.VolumeInfo{
VolumeId: "vol-123",
},
}
volumeAccessor.provisionedMachines[machine.String()] = instance.Id("already-provisioned-1")
attachmentLife := func(ids []params.MachineStorageId) ([]params.LifeResult, error) {
return []params.LifeResult{{Life: params.Dying}}, nil
}
// mockFunc's After will progress the current time by the specified
// duration and signal the channel immediately.
clock := &mockClock{}
var detachVolumeTimes []time.Time
s.provider.detachVolumesFunc = func(args []storage.VolumeAttachmentParams) ([]error, error) {
detachVolumeTimes = append(detachVolumeTimes, clock.Now())
if len(detachVolumeTimes) < 10 {
return []error{errors.New("badness")}, nil
}
return []error{nil}, nil
}
removed := make(chan interface{})
removeAttachments := func(ids []params.MachineStorageId) ([]params.ErrorResult, error) {
close(removed)
return make([]params.ErrorResult, len(ids)), nil
}
args := &workerArgs{
volumes: volumeAccessor,
clock: clock,
life: &mockLifecycleManager{
attachmentLife: attachmentLife,
removeAttachments: removeAttachments,
},
}
worker := newStorageProvisioner(c, args)
defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
defer worker.Kill()
volumeAccessor.volumesWatcher.changes <- []string{volume.Id()}
args.environ.watcher.changes <- struct{}{}
volumeAccessor.attachmentsWatcher.changes <- []params.MachineStorageId{attachmentId}
waitChannel(c, removed, "waiting for attachment to be removed")
c.Assert(detachVolumeTimes, gc.HasLen, 10)
// The first attempt should have been immediate: T0.
c.Assert(detachVolumeTimes[0], gc.Equals, time.Time{})
delays := make([]time.Duration, len(detachVolumeTimes)-1)
for i := range detachVolumeTimes[1:] {
delays[i] = detachVolumeTimes[i+1].Sub(detachVolumeTimes[i])
}
c.Assert(delays, jc.DeepEquals, []time.Duration{
30 * time.Second,
1 * time.Minute,
2 * time.Minute,
4 * time.Minute,
8 * time.Minute,
16 * time.Minute,
30 * time.Minute, // ceiling reached
30 * time.Minute,
30 * time.Minute,
})
c.Assert(args.statusSetter.args, jc.DeepEquals, []params.EntityStatusArgs{
{Tag: "volume-1", Status: "detaching", Info: "badness"}, // DetachVolumes
{Tag: "volume-1", Status: "detaching", Info: "badness"},
{Tag: "volume-1", Status: "detaching", Info: "badness"},
{Tag: "volume-1", Status: "detaching", Info: "badness"},
{Tag: "volume-1", Status: "detaching", Info: "badness"},
{Tag: "volume-1", Status: "detaching", Info: "badness"},
{Tag: "volume-1", Status: "detaching", Info: "badness"},
{Tag: "volume-1", Status: "detaching", Info: "badness"},
{Tag: "volume-1", Status: "detaching", Info: "badness"},
{Tag: "volume-1", Status: "detached", Info: ""},
})
}
func (s *storageProvisionerSuite) TestDestroyVolumesRetry(c *gc.C) {
volume := names.NewVolumeTag("1")
volumeAccessor := newMockVolumeAccessor()
volumeAccessor.provisionVolume(volume)
life := func(tags []names.Tag) ([]params.LifeResult, error) {
return []params.LifeResult{{Life: params.Dead}}, nil
}
// mockFunc's After will progress the current time by the specified
// duration and signal the channel immediately.
clock := &mockClock{}
var destroyVolumeTimes []time.Time
s.provider.destroyVolumesFunc = func(volumeIds []string) ([]error, error) {
destroyVolumeTimes = append(destroyVolumeTimes, clock.Now())
if len(destroyVolumeTimes) < 10 {
return []error{errors.New("badness")}, nil
}
return []error{nil}, nil
}
removedChan := make(chan interface{}, 1)
remove := func(tags []names.Tag) ([]params.ErrorResult, error) {
removedChan <- tags
return make([]params.ErrorResult, len(tags)), nil
}
args := &workerArgs{
volumes: volumeAccessor,
clock: clock,
life: &mockLifecycleManager{
life: life,
remove: remove,
},
}
worker := newStorageProvisioner(c, args)
defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
defer worker.Kill()
volumeAccessor.volumesWatcher.changes <- []string{volume.Id()}
args.environ.watcher.changes <- struct{}{}
waitChannel(c, removedChan, "waiting for volume to be removed")
c.Assert(destroyVolumeTimes, gc.HasLen, 10)
// The first attempt should have been immediate: T0.
c.Assert(destroyVolumeTimes[0], gc.Equals, time.Time{})
delays := make([]time.Duration, len(destroyVolumeTimes)-1)
for i := range destroyVolumeTimes[1:] {
delays[i] = destroyVolumeTimes[i+1].Sub(destroyVolumeTimes[i])
}
c.Assert(delays, jc.DeepEquals, []time.Duration{
30 * time.Second,
1 * time.Minute,
2 * time.Minute,
4 * time.Minute,
8 * time.Minute,
16 * time.Minute,
30 * time.Minute, // ceiling reached
30 * time.Minute,
30 * time.Minute,
})
c.Assert(args.statusSetter.args, jc.DeepEquals, []params.EntityStatusArgs{
{Tag: "volume-1", Status: "destroying", Info: "badness"},
{Tag: "volume-1", Status: "destroying", Info: "badness"},
{Tag: "volume-1", Status: "destroying", Info: "badness"},
{Tag: "volume-1", Status: "destroying", Info: "badness"},
{Tag: "volume-1", Status: "destroying", Info: "badness"},
{Tag: "volume-1", Status: "destroying", Info: "badness"},
{Tag: "volume-1", Status: "destroying", Info: "badness"},
{Tag: "volume-1", Status: "destroying", Info: "badness"},
{Tag: "volume-1", Status: "destroying", Info: "badness"},
})
}