Exemplo n.º 1
0
func (s *storageProvisionerSuite) TestAttachVolumeRetry(c *gc.C) {
	volumeInfoSet := make(chan interface{})
	volumeAccessor := newMockVolumeAccessor()
	volumeAccessor.provisionedMachines["machine-1"] = instance.Id("already-provisioned-1")
	volumeAccessor.setVolumeInfo = func(volumes []params.Volume) ([]params.ErrorResult, error) {
		defer close(volumeInfoSet)
		return make([]params.ErrorResult, len(volumes)), nil
	}
	volumeAttachmentInfoSet := make(chan interface{})
	volumeAccessor.setVolumeAttachmentInfo = func(volumeAttachments []params.VolumeAttachment) ([]params.ErrorResult, error) {
		defer close(volumeAttachmentInfoSet)
		return make([]params.ErrorResult, len(volumeAttachments)), nil
	}

	// mockFunc's After will progress the current time by the specified
	// duration and signal the channel immediately.
	clock := &mockClock{}
	var attachVolumeTimes []time.Time

	s.provider.attachVolumesFunc = func(args []storage.VolumeAttachmentParams) ([]storage.AttachVolumesResult, error) {
		attachVolumeTimes = append(attachVolumeTimes, clock.Now())
		if len(attachVolumeTimes) < 10 {
			return []storage.AttachVolumesResult{{Error: errors.New("badness")}}, nil
		}
		return []storage.AttachVolumesResult{{
			VolumeAttachment: &storage.VolumeAttachment{
				args[0].Volume,
				args[0].Machine,
				storage.VolumeAttachmentInfo{
					DeviceName: "/dev/sda1",
				},
			},
		}}, nil
	}

	args := &workerArgs{volumes: volumeAccessor, clock: clock}
	worker := newStorageProvisioner(c, args)
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()

	volumeAccessor.attachmentsWatcher.changes <- []params.MachineStorageId{{
		MachineTag: "machine-1", AttachmentTag: "volume-1",
	}}
	volumeAccessor.volumesWatcher.changes <- []string{"1"}
	args.environ.watcher.changes <- struct{}{}
	waitChannel(c, volumeInfoSet, "waiting for volume info to be set")
	waitChannel(c, volumeAttachmentInfoSet, "waiting for volume attachments to be set")
	c.Assert(attachVolumeTimes, gc.HasLen, 10)

	// The first attempt should have been immediate: T0.
	c.Assert(attachVolumeTimes[0], gc.Equals, time.Time{})

	delays := make([]time.Duration, len(attachVolumeTimes)-1)
	for i := range attachVolumeTimes[1:] {
		delays[i] = attachVolumeTimes[i+1].Sub(attachVolumeTimes[i])
	}
	c.Assert(delays, jc.DeepEquals, []time.Duration{
		30 * time.Second,
		1 * time.Minute,
		2 * time.Minute,
		4 * time.Minute,
		8 * time.Minute,
		16 * time.Minute,
		30 * time.Minute, // ceiling reached
		30 * time.Minute,
		30 * time.Minute,
	})

	c.Assert(args.statusSetter.args, jc.DeepEquals, []params.EntityStatusArgs{
		{Tag: "volume-1", Status: "attaching", Info: ""},        // CreateVolumes
		{Tag: "volume-1", Status: "attaching", Info: "badness"}, // AttachVolumes
		{Tag: "volume-1", Status: "attaching", Info: "badness"},
		{Tag: "volume-1", Status: "attaching", Info: "badness"},
		{Tag: "volume-1", Status: "attaching", Info: "badness"},
		{Tag: "volume-1", Status: "attaching", Info: "badness"},
		{Tag: "volume-1", Status: "attaching", Info: "badness"},
		{Tag: "volume-1", Status: "attaching", Info: "badness"},
		{Tag: "volume-1", Status: "attaching", Info: "badness"},
		{Tag: "volume-1", Status: "attaching", Info: "badness"},
		{Tag: "volume-1", Status: "attached", Info: ""},
	})
}
Exemplo n.º 2
0
func (s *storageProvisionerSuite) TestDetachVolumesRetry(c *gc.C) {
	machine := names.NewMachineTag("1")
	volume := names.NewVolumeTag("1")
	attachmentId := params.MachineStorageId{
		MachineTag: machine.String(), AttachmentTag: volume.String(),
	}
	volumeAccessor := newMockVolumeAccessor()
	volumeAccessor.provisionedAttachments[attachmentId] = params.VolumeAttachment{
		MachineTag: machine.String(),
		VolumeTag:  volume.String(),
	}
	volumeAccessor.provisionedVolumes[volume.String()] = params.Volume{
		VolumeTag: volume.String(),
		Info: params.VolumeInfo{
			VolumeId: "vol-123",
		},
	}
	volumeAccessor.provisionedMachines[machine.String()] = instance.Id("already-provisioned-1")

	attachmentLife := func(ids []params.MachineStorageId) ([]params.LifeResult, error) {
		return []params.LifeResult{{Life: params.Dying}}, nil
	}

	// mockFunc's After will progress the current time by the specified
	// duration and signal the channel immediately.
	clock := &mockClock{}
	var detachVolumeTimes []time.Time

	s.provider.detachVolumesFunc = func(args []storage.VolumeAttachmentParams) ([]error, error) {
		detachVolumeTimes = append(detachVolumeTimes, clock.Now())
		if len(detachVolumeTimes) < 10 {
			return []error{errors.New("badness")}, nil
		}
		return []error{nil}, nil
	}

	removed := make(chan interface{})
	removeAttachments := func(ids []params.MachineStorageId) ([]params.ErrorResult, error) {
		close(removed)
		return make([]params.ErrorResult, len(ids)), nil
	}

	args := &workerArgs{
		volumes: volumeAccessor,
		clock:   clock,
		life: &mockLifecycleManager{
			attachmentLife:    attachmentLife,
			removeAttachments: removeAttachments,
		},
	}
	worker := newStorageProvisioner(c, args)
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()

	volumeAccessor.volumesWatcher.changes <- []string{volume.Id()}
	args.environ.watcher.changes <- struct{}{}
	volumeAccessor.attachmentsWatcher.changes <- []params.MachineStorageId{attachmentId}
	waitChannel(c, removed, "waiting for attachment to be removed")
	c.Assert(detachVolumeTimes, gc.HasLen, 10)

	// The first attempt should have been immediate: T0.
	c.Assert(detachVolumeTimes[0], gc.Equals, time.Time{})

	delays := make([]time.Duration, len(detachVolumeTimes)-1)
	for i := range detachVolumeTimes[1:] {
		delays[i] = detachVolumeTimes[i+1].Sub(detachVolumeTimes[i])
	}
	c.Assert(delays, jc.DeepEquals, []time.Duration{
		30 * time.Second,
		1 * time.Minute,
		2 * time.Minute,
		4 * time.Minute,
		8 * time.Minute,
		16 * time.Minute,
		30 * time.Minute, // ceiling reached
		30 * time.Minute,
		30 * time.Minute,
	})

	c.Assert(args.statusSetter.args, jc.DeepEquals, []params.EntityStatusArgs{
		{Tag: "volume-1", Status: "detaching", Info: "badness"}, // DetachVolumes
		{Tag: "volume-1", Status: "detaching", Info: "badness"},
		{Tag: "volume-1", Status: "detaching", Info: "badness"},
		{Tag: "volume-1", Status: "detaching", Info: "badness"},
		{Tag: "volume-1", Status: "detaching", Info: "badness"},
		{Tag: "volume-1", Status: "detaching", Info: "badness"},
		{Tag: "volume-1", Status: "detaching", Info: "badness"},
		{Tag: "volume-1", Status: "detaching", Info: "badness"},
		{Tag: "volume-1", Status: "detaching", Info: "badness"},
		{Tag: "volume-1", Status: "detached", Info: ""},
	})
}
Exemplo n.º 3
0
func (s *storageProvisionerSuite) TestDestroyVolumesRetry(c *gc.C) {
	volume := names.NewVolumeTag("1")
	volumeAccessor := newMockVolumeAccessor()
	volumeAccessor.provisionVolume(volume)

	life := func(tags []names.Tag) ([]params.LifeResult, error) {
		return []params.LifeResult{{Life: params.Dead}}, nil
	}

	// mockFunc's After will progress the current time by the specified
	// duration and signal the channel immediately.
	clock := &mockClock{}
	var destroyVolumeTimes []time.Time

	s.provider.destroyVolumesFunc = func(volumeIds []string) ([]error, error) {
		destroyVolumeTimes = append(destroyVolumeTimes, clock.Now())
		if len(destroyVolumeTimes) < 10 {
			return []error{errors.New("badness")}, nil
		}
		return []error{nil}, nil
	}

	removedChan := make(chan interface{}, 1)
	remove := func(tags []names.Tag) ([]params.ErrorResult, error) {
		removedChan <- tags
		return make([]params.ErrorResult, len(tags)), nil
	}

	args := &workerArgs{
		volumes: volumeAccessor,
		clock:   clock,
		life: &mockLifecycleManager{
			life:   life,
			remove: remove,
		},
	}
	worker := newStorageProvisioner(c, args)
	defer func() { c.Assert(worker.Wait(), gc.IsNil) }()
	defer worker.Kill()

	volumeAccessor.volumesWatcher.changes <- []string{volume.Id()}
	args.environ.watcher.changes <- struct{}{}
	waitChannel(c, removedChan, "waiting for volume to be removed")
	c.Assert(destroyVolumeTimes, gc.HasLen, 10)

	// The first attempt should have been immediate: T0.
	c.Assert(destroyVolumeTimes[0], gc.Equals, time.Time{})

	delays := make([]time.Duration, len(destroyVolumeTimes)-1)
	for i := range destroyVolumeTimes[1:] {
		delays[i] = destroyVolumeTimes[i+1].Sub(destroyVolumeTimes[i])
	}
	c.Assert(delays, jc.DeepEquals, []time.Duration{
		30 * time.Second,
		1 * time.Minute,
		2 * time.Minute,
		4 * time.Minute,
		8 * time.Minute,
		16 * time.Minute,
		30 * time.Minute, // ceiling reached
		30 * time.Minute,
		30 * time.Minute,
	})

	c.Assert(args.statusSetter.args, jc.DeepEquals, []params.EntityStatusArgs{
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
		{Tag: "volume-1", Status: "destroying", Info: "badness"},
	})
}