func detachFilesystems(ctx *context, attachments []storage.FilesystemAttachmentParams) error {
paramsBySource, filesystemSources, err := filesystemAttachmentParamsBySource(ctx, attachments)
if err != nil {
return errors.Trace(err)
}
for sourceName, params := range paramsBySource {
logger.Debugf("detaching filesystems: %v", params)
filesystemSource := filesystemSources[sourceName]
results, err := filesystemSource.DetachFilesystems(params)
if err != nil {
return errors.Annotatef(err, "detaching filesystems from source %q", sourceName)
}
for i, err := range results {
if err == nil {
continue
}
return errors.Annotatef(
err, "detaching %s from %s",
names.ReadableString(params[i].Filesystem),
names.ReadableString(params[i].Machine),
)
}
}
return nil
}
// attachVolumes creates volume attachments with the specified parameters.
func attachVolumes(ctx *context, ops map[params.MachineStorageId]*attachVolumeOp) error {
volumeAttachmentParams := make([]storage.VolumeAttachmentParams, 0, len(ops))
for _, op := range ops {
volumeAttachmentParams = append(volumeAttachmentParams, op.args)
}
paramsBySource, volumeSources, err := volumeAttachmentParamsBySource(
ctx.environConfig, ctx.config.StorageDir, volumeAttachmentParams,
)
if err != nil {
return errors.Trace(err)
}
var reschedule []scheduleOp
var volumeAttachments []storage.VolumeAttachment
var statuses []params.EntityStatusArgs
for sourceName, volumeAttachmentParams := range paramsBySource {
logger.Debugf("attaching volumes: %+v", volumeAttachmentParams)
volumeSource := volumeSources[sourceName]
results, err := volumeSource.AttachVolumes(volumeAttachmentParams)
if err != nil {
return errors.Annotatef(err, "attaching volumes from source %q", sourceName)
}
for i, result := range results {
p := volumeAttachmentParams[i]
statuses = append(statuses, params.EntityStatusArgs{
Tag: p.Volume.String(),
Status: params.StatusAttached,
})
status := &statuses[len(statuses)-1]
if result.Error != nil {
// Reschedule the volume attachment.
id := params.MachineStorageId{
MachineTag: p.Machine.String(),
AttachmentTag: p.Volume.String(),
}
reschedule = append(reschedule, ops[id])
// Note: we keep the status as "attaching" to
// indicate that we will retry. When we distinguish
// between transient and permanent errors, we will
// set the status to "error" for permanent errors.
status.Status = params.StatusAttaching
status.Info = result.Error.Error()
logger.Debugf(
"failed to attach %s to %s: %v",
names.ReadableString(p.Volume),
names.ReadableString(p.Machine),
result.Error,
)
continue
}
volumeAttachments = append(volumeAttachments, *result.VolumeAttachment)
}
}
scheduleOperations(ctx, reschedule...)
setStatus(ctx, statuses)
if err := setVolumeAttachmentInfo(ctx, volumeAttachments); err != nil {
return errors.Trace(err)
}
return nil
}
// createFilesystemAttachments creates filesystem attachments with the specified parameters.
func createFilesystemAttachments(
ctx *context,
params []storage.FilesystemAttachmentParams,
) ([]storage.FilesystemAttachment, error) {
paramsBySource, filesystemSources, err := filesystemAttachmentParamsBySource(ctx, params)
if err != nil {
return nil, errors.Trace(err)
}
var allFilesystemAttachments []storage.FilesystemAttachment
for sourceName, params := range paramsBySource {
logger.Debugf("attaching filesystems: %v", params)
filesystemSource := filesystemSources[sourceName]
results, err := filesystemSource.AttachFilesystems(params)
if err != nil {
return nil, errors.Annotatef(err, "attaching filesystems from source %q", sourceName)
}
for i, result := range results {
if result.Error != nil {
return nil, errors.Annotatef(
err, "attaching %s to %s",
names.ReadableString(params[i].Filesystem),
names.ReadableString(params[i].Machine),
)
}
allFilesystemAttachments = append(allFilesystemAttachments, *result.FilesystemAttachment)
}
}
return allFilesystemAttachments, nil
}
func setVolumeAttachmentInfo(ctx *context, volumeAttachments []storage.VolumeAttachment) error {
if len(volumeAttachments) == 0 {
return nil
}
// TODO(axw) we need to be able to list volume attachments in the
// provider, by environment, so that we can "harvest" them if they're
// unknown. This will take care of killing volumes that we fail to
// record in state.
errorResults, err := ctx.config.Volumes.SetVolumeAttachmentInfo(
volumeAttachmentsFromStorage(volumeAttachments),
)
if err != nil {
return errors.Annotate(err, "publishing volumes to state")
}
for i, result := range errorResults {
if result.Error != nil {
return errors.Annotatef(
result.Error, "publishing attachment of %s to %s to state",
names.ReadableString(volumeAttachments[i].Volume),
names.ReadableString(volumeAttachments[i].Machine),
)
}
// Record the volume attachment in the context.
id := params.MachineStorageId{
MachineTag: volumeAttachments[i].Machine.String(),
AttachmentTag: volumeAttachments[i].Volume.String(),
}
ctx.volumeAttachments[id] = volumeAttachments[i]
removePendingVolumeAttachment(ctx, id)
}
return nil
}
func setFilesystemAttachmentInfo(ctx *context, filesystemAttachments []storage.FilesystemAttachment) error {
if len(filesystemAttachments) == 0 {
return nil
}
// TODO(axw) we need to be able to list filesystem attachments in the
// provider, by environment, so that we can "harvest" them if they're
// unknown. This will take care of killing filesystems that we fail to
// record in state.
errorResults, err := ctx.filesystemAccessor.SetFilesystemAttachmentInfo(
filesystemAttachmentsFromStorage(filesystemAttachments),
)
if err != nil {
return errors.Annotate(err, "publishing filesystems to state")
}
for i, result := range errorResults {
if result.Error != nil {
return errors.Annotatef(
result.Error, "publishing attachment of %s to %s to state",
names.ReadableString(filesystemAttachments[i].Filesystem),
names.ReadableString(filesystemAttachments[i].Machine),
)
}
// Record the filesystem attachment in the context.
ctx.filesystemAttachments[params.MachineStorageId{
MachineTag: filesystemAttachments[i].Machine.String(),
AttachmentTag: filesystemAttachments[i].Filesystem.String(),
}] = filesystemAttachments[i]
}
return nil
}
func (s *StatusSetter) updateEntityStatusData(tag names.Tag, data map[string]interface{}) error {
entity0, err := s.st.FindEntity(tag)
if err != nil {
return err
}
statusGetter, ok := entity0.(status.StatusGetter)
if !ok {
return NotSupportedError(tag, "getting status")
}
existingStatusInfo, err := statusGetter.Status()
if err != nil {
return err
}
newData := existingStatusInfo.Data
if newData == nil {
newData = data
} else {
for k, v := range data {
newData[k] = v
}
}
entity, ok := entity0.(status.StatusSetter)
if !ok {
return NotSupportedError(tag, "updating status")
}
if len(newData) > 0 && existingStatusInfo.Status != status.StatusError {
return fmt.Errorf("%s is not in an error state", names.ReadableString(tag))
}
return entity.SetStatus(existingStatusInfo.Status, existingStatusInfo.Message, newData)
}
func (f *fakeRetryProvisioningClient) RetryProvisioning(machines ...names.MachineTag) (
[]params.ErrorResult, error) {
if f.err != nil {
return nil, f.err
}
results := make([]params.ErrorResult, len(machines))
// For each of the machines passed in, verify that we have the
// id and that the info string is "broken".
for i, machine := range machines {
m, ok := f.m[machine.Id()]
if ok {
if m.info == "broken" {
// The real RetryProvisioning command sets the
// status data "transient" : true.
m.data["transient"] = true
} else {
results[i].Error = common.ServerError(
fmt.Errorf("%s is not in an error state",
names.ReadableString(machine)))
}
} else {
results[i].Error = common.ServerError(
errors.NotFoundf("machine %s", machine.Id()))
}
}
return results, nil
}
// storageChanged responds to unit storage changes.
func (w *RemoteStateWatcher) storageChanged(keys []string) error {
tags := make([]names.StorageTag, len(keys))
for i, key := range keys {
tags[i] = names.NewStorageTag(key)
}
ids := make([]params.StorageAttachmentId, len(keys))
for i, tag := range tags {
ids[i] = params.StorageAttachmentId{
StorageTag: tag.String(),
UnitTag: w.unit.Tag().String(),
}
}
results, err := w.st.StorageAttachmentLife(ids)
if err != nil {
return errors.Trace(err)
}
w.mu.Lock()
defer w.mu.Unlock()
for i, result := range results {
tag := tags[i]
if result.Error == nil {
if storageSnapshot, ok := w.current.Storage[tag]; ok {
// We've previously started a watcher for this storage
// attachment, so all we needed to do was update the
// lifecycle state.
storageSnapshot.Life = result.Life
w.current.Storage[tag] = storageSnapshot
continue
}
// We haven't seen this storage attachment before, so start
// a watcher now; add it to our catacomb in case of mishap;
// and wait for the initial event.
saw, err := w.st.WatchStorageAttachment(tag, w.unit.Tag())
if err != nil {
return errors.Annotate(err, "watching storage attachment")
}
if err := w.catacomb.Add(saw); err != nil {
return errors.Trace(err)
}
if err := w.watchStorageAttachment(tag, result.Life, saw); err != nil {
return errors.Trace(err)
}
} else if params.IsCodeNotFound(result.Error) {
if watcher, ok := w.storageAttachmentWatchers[tag]; ok {
// already under catacomb management, any error tracked already
worker.Stop(watcher)
delete(w.storageAttachmentWatchers, tag)
}
delete(w.current.Storage, tag)
} else {
return errors.Annotatef(
result.Error, "getting life of %s attachment",
names.ReadableString(tag),
)
}
}
return nil
}
// createFilesystems creates filesystems with the specified parameters.
func createFilesystems(ctx *context, params []storage.FilesystemParams) ([]storage.Filesystem, error) {
// TODO(axw) later we may have multiple instantiations (sources)
// for a storage provider, e.g. multiple Ceph installations. For
// now we assume a single source for each provider type, with no
// configuration.
// Create filesystem sources.
filesystemSources := make(map[string]storage.FilesystemSource)
for _, params := range params {
sourceName := string(params.Provider)
if _, ok := filesystemSources[sourceName]; ok {
continue
}
if params.Volume != (names.VolumeTag{}) {
filesystemSources[sourceName] = ctx.managedFilesystemSource
continue
}
filesystemSource, err := filesystemSource(
ctx.environConfig, ctx.storageDir, sourceName, params.Provider,
)
if err != nil {
return nil, errors.Annotate(err, "getting filesystem source")
}
filesystemSources[sourceName] = filesystemSource
}
// Validate and gather filesystem parameters.
paramsBySource := make(map[string][]storage.FilesystemParams)
for _, params := range params {
sourceName := string(params.Provider)
filesystemSource := filesystemSources[sourceName]
err := filesystemSource.ValidateFilesystemParams(params)
if err != nil {
// TODO(axw) we should set an error status for params.Tag
// here, and we should retry periodically.
logger.Errorf("ignoring invalid filesystem: %v", err)
continue
}
paramsBySource[sourceName] = append(paramsBySource[sourceName], params)
}
var allFilesystems []storage.Filesystem
for sourceName, params := range paramsBySource {
logger.Debugf("creating filesystems: %v", params)
filesystemSource := filesystemSources[sourceName]
results, err := filesystemSource.CreateFilesystems(params)
if err != nil {
return nil, errors.Annotatef(err, "creating filesystems from source %q", sourceName)
}
for i, result := range results {
if result.Error != nil {
return nil, errors.Annotatef(result.Error, "creating %s", names.ReadableString(params[i].Tag))
}
allFilesystems = append(allFilesystems, *result.Filesystem)
}
}
return allFilesystems, nil
}
// AttachFilesystems is defined on the FilesystemSource interface.
func (s *rootfsFilesystemSource) AttachFilesystems(args []storage.FilesystemAttachmentParams) ([]storage.FilesystemAttachment, error) {
attachments := make([]storage.FilesystemAttachment, len(args))
for i, arg := range args {
attachment, err := s.attachFilesystem(arg)
if err != nil {
return nil, errors.Annotatef(err, "attaching %s", names.ReadableString(arg.Filesystem))
}
attachments[i] = attachment
}
return attachments, nil
}
// removeEntities removes each specified Dead entity from state.
func removeEntities(ctx *context, tags []names.Tag) error {
logger.Debugf("removing entities: %v", tags)
errorResults, err := ctx.life.Remove(tags)
if err != nil {
return errors.Annotate(err, "removing storage entities")
}
for i, result := range errorResults {
if result.Error != nil {
return errors.Annotatef(result.Error, "removing %s from state", names.ReadableString(tags[i]))
}
}
return nil
}
func volumeStorageAttachmentInfo(
st StorageInterface,
storageInstance state.StorageInstance,
machineTag names.MachineTag,
) (*storage.StorageAttachmentInfo, error) {
storageTag := storageInstance.StorageTag()
volume, err := st.StorageInstanceVolume(storageTag)
if err != nil {
return nil, errors.Annotate(err, "getting volume")
}
volumeInfo, err := volume.Info()
if err != nil {
return nil, errors.Annotate(err, "getting volume info")
}
volumeAttachment, err := st.VolumeAttachment(machineTag, volume.VolumeTag())
if err != nil {
return nil, errors.Annotate(err, "getting volume attachment")
}
volumeAttachmentInfo, err := volumeAttachment.Info()
if err != nil {
return nil, errors.Annotate(err, "getting volume attachment info")
}
blockDevices, err := st.BlockDevices(machineTag)
if err != nil {
return nil, errors.Annotate(err, "getting block devices")
}
blockDevice, ok := MatchingBlockDevice(
blockDevices,
volumeInfo,
volumeAttachmentInfo,
)
if !ok {
// We must not say that a block-kind storage attachment is
// provisioned until its block device has shown up on the
// machine, otherwise the charm may attempt to use it and
// fail.
return nil, errors.NotProvisionedf("%v", names.ReadableString(storageTag))
}
devicePath, err := volumeAttachmentDevicePath(
volumeInfo,
volumeAttachmentInfo,
*blockDevice,
)
if err != nil {
return nil, errors.Trace(err)
}
return &storage.StorageAttachmentInfo{
storage.StorageKindBlock,
devicePath,
}, nil
}
// processDeadFilesystems processes the FilesystemResults for Dead filesystems,
// deprovisioning filesystems and removing from state as necessary.
func processDeadFilesystems(ctx *context, tags []names.FilesystemTag, filesystemResults []params.FilesystemResult) error {
for _, tag := range tags {
delete(ctx.pendingFilesystems, tag)
}
var destroy []names.FilesystemTag
var remove []names.Tag
for i, result := range filesystemResults {
tag := tags[i]
if result.Error == nil {
logger.Debugf("filesystem %s is provisioned, queuing for deprovisioning", tag.Id())
filesystem, err := filesystemFromParams(result.Result)
if err != nil {
return errors.Annotate(err, "getting filesystem info")
}
ctx.filesystems[tag] = filesystem
destroy = append(destroy, tag)
continue
}
if params.IsCodeNotProvisioned(result.Error) {
logger.Debugf("filesystem %s is not provisioned, queuing for removal", tag.Id())
remove = append(remove, tag)
continue
}
return errors.Annotatef(result.Error, "getting filesystem information for filesystem %s", tag.Id())
}
if len(destroy)+len(remove) == 0 {
return nil
}
if len(destroy) > 0 {
errorResults, err := destroyFilesystems(ctx, destroy)
if err != nil {
return errors.Annotate(err, "destroying filesystems")
}
for i, tag := range destroy {
if err := errorResults[i]; err != nil {
return errors.Annotatef(err, "destroying %s", names.ReadableString(tag))
}
remove = append(remove, tag)
}
}
if err := removeEntities(ctx, remove); err != nil {
return errors.Annotate(err, "removing filesystems from state")
}
return nil
}
func (w *upgradesteps) prepareForUpgrade() (*state.UpgradeInfo, error) {
logger.Infof("checking that upgrade can proceed")
if err := w.preUpgradeSteps(w.st, w.agent.CurrentConfig(), w.st != nil, w.isMaster); err != nil {
return nil, errors.Annotatef(err, "%s cannot be upgraded", names.ReadableString(w.tag))
}
if !w.isStateServer {
return nil, nil
}
logger.Infof("signalling that this state server is ready for upgrade")
info, err := w.st.EnsureUpgradeInfo(w.tag.Id(), w.fromVersion, w.toVersion)
if err != nil {
return nil, errors.Trace(err)
}
// State servers need to wait for other state servers to be ready
// to run the upgrade steps.
logger.Infof("waiting for other state servers to be ready for upgrade")
if err := w.waitForOtherStateServers(info); err != nil {
if err == tomb.ErrDying {
logger.Warningf(`stopped waiting for other state servers: %v`, err)
return nil, err
}
logger.Errorf(`aborted wait for other state servers: %v`, err)
// If master, trigger a rollback to the previous agent version.
if w.isMaster {
logger.Errorf("downgrading environment agent version to %v due to aborted upgrade",
w.fromVersion)
if rollbackErr := w.st.SetEnvironAgentVersion(w.fromVersion); rollbackErr != nil {
logger.Errorf("rollback failed: %v", rollbackErr)
return nil, errors.Annotate(rollbackErr, "failed to roll back desired agent version")
}
}
return nil, errors.Annotate(err, "aborted wait for other state servers")
}
if w.isMaster {
logger.Infof("finished waiting - all state servers are ready to run upgrade steps")
} else {
logger.Infof("finished waiting - the master has completed its upgrade steps")
}
return info, nil
}
func (*tagSuite) TestReadableString(c *gc.C) {
var readableStringTests = []struct {
tag names.Tag
result string
}{{
tag: nil,
result: "",
}, {
tag: names.NewMachineTag("0"),
result: "machine 0",
}, {
tag: names.NewUnitTag("wordpress/2"),
result: "unit wordpress/2",
}}
for i, test := range readableStringTests {
c.Logf("test %d: expected result %q", i, test.result)
resultStr := names.ReadableString(test.tag)
c.Assert(resultStr, gc.Equals, test.result)
}
}
func createVolumeDetailsList(
st storageAccess,
volumes []state.Volume,
attachments map[names.VolumeTag][]state.VolumeAttachment,
) ([]params.VolumeDetails, error) {
if len(volumes) == 0 {
return nil, nil
}
results := make([]params.VolumeDetails, len(volumes))
for i, v := range volumes {
details, err := createVolumeDetails(st, v, attachments[v.VolumeTag()])
if err != nil {
return nil, errors.Annotatef(
err, "getting details for %s",
names.ReadableString(v.VolumeTag()),
)
}
results[i] = *details
}
return results, nil
}
func createFilesystemDetailsList(
st storageAccess,
filesystems []state.Filesystem,
attachments map[names.FilesystemTag][]state.FilesystemAttachment,
) ([]params.FilesystemDetails, error) {
if len(filesystems) == 0 {
return nil, nil
}
results := make([]params.FilesystemDetails, len(filesystems))
for i, f := range filesystems {
details, err := createFilesystemDetails(st, f, attachments[f.FilesystemTag()])
if err != nil {
return nil, errors.Annotatef(
err, "getting details for %s",
names.ReadableString(f.FilesystemTag()),
)
}
results[i] = *details
}
return results, nil
}
// storageEntityLife queries the lifecycle state of each specified
// storage entity (volume or filesystem), and then partitions the
// tags by them.
func storageEntityLife(ctx *context, tags []names.Tag) (alive, dying, dead []names.Tag, _ error) {
lifeResults, err := ctx.life.Life(tags)
if err != nil {
return nil, nil, nil, errors.Annotate(err, "getting storage entity life")
}
for i, result := range lifeResults {
if result.Error != nil {
return nil, nil, nil, errors.Annotatef(
result.Error, "getting life of %s",
names.ReadableString(tags[i]),
)
}
switch result.Life {
case params.Alive:
alive = append(alive, tags[i])
case params.Dying:
dying = append(dying, tags[i])
case params.Dead:
dead = append(dead, tags[i])
}
}
return alive, dying, dead, nil
}
func (api *API) listStorageDetails(filter params.StorageFilter) ([]params.StorageDetails, error) {
if filter != (params.StorageFilter{}) {
// StorageFilter has no fields at the time of writing, but
// check that no fields are set in case we forget to update
// this code.
return nil, errors.NotSupportedf("storage filters")
}
stateInstances, err := api.storage.AllStorageInstances()
if err != nil {
return nil, common.ServerError(err)
}
results := make([]params.StorageDetails, len(stateInstances))
for i, stateInstance := range stateInstances {
details, err := createStorageDetails(api.storage, stateInstance)
if err != nil {
return nil, errors.Annotatef(
err, "getting details for %s",
names.ReadableString(stateInstance.Tag()),
)
}
results[i] = *details
}
return results, nil
}
// createFilesystems creates filesystems with the specified parameters.
func createFilesystems(ctx *context, ops map[names.FilesystemTag]*createFilesystemOp) error {
filesystemParams := make([]storage.FilesystemParams, 0, len(ops))
for _, op := range ops {
filesystemParams = append(filesystemParams, op.args)
}
paramsBySource, filesystemSources, err := filesystemParamsBySource(
ctx.environConfig, ctx.storageDir,
filesystemParams, ctx.managedFilesystemSource,
)
if err != nil {
return errors.Trace(err)
}
var reschedule []scheduleOp
var filesystems []storage.Filesystem
var statuses []params.EntityStatusArgs
for sourceName, filesystemParams := range paramsBySource {
logger.Debugf("creating filesystems: %v", filesystemParams)
filesystemSource := filesystemSources[sourceName]
validFilesystemParams, validationErrors := validateFilesystemParams(
filesystemSource, filesystemParams,
)
for i, err := range validationErrors {
if err == nil {
continue
}
statuses = append(statuses, params.EntityStatusArgs{
Tag: filesystemParams[i].Tag.String(),
Status: params.StatusError,
Info: err.Error(),
})
logger.Debugf(
"failed to validate parameters for %s: %v",
names.ReadableString(filesystemParams[i].Tag), err,
)
}
filesystemParams = validFilesystemParams
if len(filesystemParams) == 0 {
continue
}
results, err := filesystemSource.CreateFilesystems(filesystemParams)
if err != nil {
return errors.Annotatef(err, "creating filesystems from source %q", sourceName)
}
for i, result := range results {
statuses = append(statuses, params.EntityStatusArgs{
Tag: filesystemParams[i].Tag.String(),
Status: params.StatusAttaching,
})
status := &statuses[len(statuses)-1]
if result.Error != nil {
// Reschedule the filesystem creation.
reschedule = append(reschedule, ops[filesystemParams[i].Tag])
// Note: we keep the status as "pending" to indicate
// that we will retry. When we distinguish between
// transient and permanent errors, we will set the
// status to "error" for permanent errors.
status.Status = params.StatusPending
status.Info = result.Error.Error()
logger.Debugf(
"failed to create %s: %v",
names.ReadableString(filesystemParams[i].Tag),
result.Error,
)
continue
}
filesystems = append(filesystems, *result.Filesystem)
}
}
scheduleOperations(ctx, reschedule...)
setStatus(ctx, statuses)
if len(filesystems) == 0 {
return nil
}
// TODO(axw) we need to be able to list filesystems in the provider,
// by environment, so that we can "harvest" them if they're
// unknown. This will take care of killing filesystems that we fail
// to record in state.
errorResults, err := ctx.filesystemAccessor.SetFilesystemInfo(filesystemsFromStorage(filesystems))
if err != nil {
return errors.Annotate(err, "publishing filesystems to state")
}
for i, result := range errorResults {
if result.Error != nil {
logger.Errorf(
"publishing filesystem %s to state: %v",
filesystems[i].Tag.Id(),
result.Error,
)
}
}
for _, v := range filesystems {
updateFilesystem(ctx, v)
}
return nil
}
func (s *baseStorageSuite) constructState() *mockState {
s.unitTag = names.NewUnitTag("mysql/0")
s.storageTag = names.NewStorageTag("data/0")
s.storageInstance = &mockStorageInstance{
kind: state.StorageKindFilesystem,
owner: s.unitTag,
storageTag: s.storageTag,
}
storageInstanceAttachment := &mockStorageAttachment{storage: s.storageInstance}
s.machineTag = names.NewMachineTag("66")
s.filesystemTag = names.NewFilesystemTag("104")
s.volumeTag = names.NewVolumeTag("22")
s.filesystem = &mockFilesystem{
tag: s.filesystemTag,
storage: &s.storageTag,
}
s.filesystemAttachment = &mockFilesystemAttachment{
filesystem: s.filesystemTag,
machine: s.machineTag,
}
s.volume = &mockVolume{tag: s.volumeTag, storage: &s.storageTag}
s.volumeAttachment = &mockVolumeAttachment{
VolumeTag: s.volumeTag,
MachineTag: s.machineTag,
}
s.blocks = make(map[state.BlockType]state.Block)
return &mockState{
allStorageInstances: func() ([]state.StorageInstance, error) {
s.calls = append(s.calls, allStorageInstancesCall)
return []state.StorageInstance{s.storageInstance}, nil
},
storageInstance: func(sTag names.StorageTag) (state.StorageInstance, error) {
s.calls = append(s.calls, storageInstanceCall)
if sTag == s.storageTag {
return s.storageInstance, nil
}
return nil, errors.NotFoundf("%s", names.ReadableString(sTag))
},
storageInstanceAttachments: func(tag names.StorageTag) ([]state.StorageAttachment, error) {
s.calls = append(s.calls, storageInstanceAttachmentsCall)
if tag == s.storageTag {
return []state.StorageAttachment{storageInstanceAttachment}, nil
}
return nil, errors.NotFoundf("%s", names.ReadableString(tag))
},
storageInstanceFilesystem: func(sTag names.StorageTag) (state.Filesystem, error) {
s.calls = append(s.calls, storageInstanceFilesystemCall)
if sTag == s.storageTag {
return s.filesystem, nil
}
return nil, errors.NotFoundf("%s", names.ReadableString(sTag))
},
storageInstanceFilesystemAttachment: func(m names.MachineTag, f names.FilesystemTag) (state.FilesystemAttachment, error) {
s.calls = append(s.calls, storageInstanceFilesystemAttachmentCall)
if m == s.machineTag && f == s.filesystemTag {
return s.filesystemAttachment, nil
}
return nil, errors.NotFoundf("filesystem attachment %s:%s", m, f)
},
storageInstanceVolume: func(t names.StorageTag) (state.Volume, error) {
s.calls = append(s.calls, storageInstanceVolumeCall)
if t == s.storageTag {
return s.volume, nil
}
return nil, errors.NotFoundf("%s", names.ReadableString(t))
},
unitAssignedMachine: func(u names.UnitTag) (names.MachineTag, error) {
s.calls = append(s.calls, unitAssignedMachineCall)
if u == s.unitTag {
return s.machineTag, nil
}
return names.MachineTag{}, errors.NotFoundf("%s", names.ReadableString(u))
},
volume: func(tag names.VolumeTag) (state.Volume, error) {
s.calls = append(s.calls, volumeCall)
if tag == s.volumeTag {
return s.volume, nil
}
return nil, errors.NotFoundf("%s", names.ReadableString(tag))
},
machineVolumeAttachments: func(machine names.MachineTag) ([]state.VolumeAttachment, error) {
s.calls = append(s.calls, machineVolumeAttachmentsCall)
if machine == s.machineTag {
return []state.VolumeAttachment{s.volumeAttachment}, nil
}
return nil, nil
},
volumeAttachments: func(volume names.VolumeTag) ([]state.VolumeAttachment, error) {
s.calls = append(s.calls, volumeAttachmentsCall)
if volume == s.volumeTag {
return []state.VolumeAttachment{s.volumeAttachment}, nil
}
return nil, nil
},
allVolumes: func() ([]state.Volume, error) {
s.calls = append(s.calls, allVolumesCall)
return []state.Volume{s.volume}, nil
},
filesystem: func(tag names.FilesystemTag) (state.Filesystem, error) {
s.calls = append(s.calls, filesystemCall)
if tag == s.filesystemTag {
return s.filesystem, nil
}
return nil, errors.NotFoundf("%s", names.ReadableString(tag))
},
machineFilesystemAttachments: func(machine names.MachineTag) ([]state.FilesystemAttachment, error) {
s.calls = append(s.calls, machineFilesystemAttachmentsCall)
if machine == s.machineTag {
return []state.FilesystemAttachment{s.filesystemAttachment}, nil
}
return nil, nil
},
filesystemAttachments: func(filesystem names.FilesystemTag) ([]state.FilesystemAttachment, error) {
s.calls = append(s.calls, filesystemAttachmentsCall)
if filesystem == s.filesystemTag {
return []state.FilesystemAttachment{s.filesystemAttachment}, nil
}
return nil, nil
},
allFilesystems: func() ([]state.Filesystem, error) {
s.calls = append(s.calls, allFilesystemsCall)
return []state.Filesystem{s.filesystem}, nil
},
envName: "storagetest",
addStorageForUnit: func(u names.UnitTag, name string, cons state.StorageConstraints) error {
s.calls = append(s.calls, addStorageForUnitCall)
return nil
},
getBlockForType: func(t state.BlockType) (state.Block, bool, error) {
s.calls = append(s.calls, getBlockForTypeCall)
val, found := s.blocks[t]
return val, found, nil
},
}
}
// detachVolumes destroys volume attachments with the specified parameters.
func detachVolumes(ctx *context, ops map[params.MachineStorageId]*detachVolumeOp) error {
volumeAttachmentParams := make([]storage.VolumeAttachmentParams, 0, len(ops))
for _, op := range ops {
volumeAttachmentParams = append(volumeAttachmentParams, op.args)
}
paramsBySource, volumeSources, err := volumeAttachmentParamsBySource(
ctx.environConfig, ctx.config.StorageDir, volumeAttachmentParams,
)
if err != nil {
return errors.Trace(err)
}
var reschedule []scheduleOp
var statuses []params.EntityStatusArgs
var remove []params.MachineStorageId
for sourceName, volumeAttachmentParams := range paramsBySource {
logger.Debugf("detaching volumes: %+v", volumeAttachmentParams)
volumeSource := volumeSources[sourceName]
errs, err := volumeSource.DetachVolumes(volumeAttachmentParams)
if err != nil {
return errors.Annotatef(err, "detaching volumes from source %q", sourceName)
}
for i, err := range errs {
p := volumeAttachmentParams[i]
statuses = append(statuses, params.EntityStatusArgs{
Tag: p.Volume.String(),
// TODO(axw) when we support multiple
// attachment, we'll have to check if
// there are any other attachments
// before saying the status "detached".
Status: params.StatusDetached,
})
id := params.MachineStorageId{
MachineTag: p.Machine.String(),
AttachmentTag: p.Volume.String(),
}
status := &statuses[len(statuses)-1]
if err != nil {
reschedule = append(reschedule, ops[id])
status.Status = params.StatusDetaching
status.Info = err.Error()
logger.Debugf(
"failed to detach %s from %s: %v",
names.ReadableString(p.Volume),
names.ReadableString(p.Machine),
err,
)
continue
}
remove = append(remove, id)
}
}
scheduleOperations(ctx, reschedule...)
setStatus(ctx, statuses)
if err := removeAttachments(ctx, remove); err != nil {
return errors.Annotate(err, "removing attachments from state")
}
for _, id := range remove {
delete(ctx.volumeAttachments, id)
}
return nil
}
// destroyVolumes destroys volumes with the specified parameters.
func destroyVolumes(ctx *context, ops map[names.VolumeTag]*destroyVolumeOp) error {
tags := make([]names.VolumeTag, 0, len(ops))
for tag := range ops {
tags = append(tags, tag)
}
volumeParams, err := volumeParams(ctx, tags)
if err != nil {
return errors.Trace(err)
}
paramsBySource, volumeSources, err := volumeParamsBySource(
ctx.environConfig, ctx.config.StorageDir, volumeParams,
)
if err != nil {
return errors.Trace(err)
}
var remove []names.Tag
var reschedule []scheduleOp
var statuses []params.EntityStatusArgs
for sourceName, volumeParams := range paramsBySource {
logger.Debugf("destroying volumes from %q: %v", sourceName, volumeParams)
volumeSource := volumeSources[sourceName]
validVolumeParams, validationErrors := validateVolumeParams(volumeSource, volumeParams)
for i, err := range validationErrors {
if err == nil {
continue
}
statuses = append(statuses, params.EntityStatusArgs{
Tag: volumeParams[i].Tag.String(),
Status: params.StatusError,
Info: err.Error(),
})
logger.Debugf(
"failed to validate parameters for %s: %v",
names.ReadableString(volumeParams[i].Tag), err,
)
}
volumeParams = validVolumeParams
if len(volumeParams) == 0 {
continue
}
volumeIds := make([]string, len(volumeParams))
for i, volumeParams := range volumeParams {
volume, ok := ctx.volumes[volumeParams.Tag]
if !ok {
return errors.NotFoundf("volume %s", volumeParams.Tag.Id())
}
volumeIds[i] = volume.VolumeId
}
errs, err := volumeSource.DestroyVolumes(volumeIds)
if err != nil {
return errors.Trace(err)
}
for i, err := range errs {
tag := volumeParams[i].Tag
if err == nil {
remove = append(remove, tag)
continue
}
// Failed to destroy volume; reschedule and update status.
reschedule = append(reschedule, ops[tag])
statuses = append(statuses, params.EntityStatusArgs{
Tag: tag.String(),
Status: params.StatusDestroying,
Info: err.Error(),
})
}
}
scheduleOperations(ctx, reschedule...)
setStatus(ctx, statuses)
if err := removeEntities(ctx, remove); err != nil {
return errors.Annotate(err, "removing volumes from state")
}
return nil
}
// createVolumes creates volumes with the specified parameters.
func createVolumes(ctx *context, ops map[names.VolumeTag]*createVolumeOp) error {
volumeParams := make([]storage.VolumeParams, 0, len(ops))
for _, op := range ops {
volumeParams = append(volumeParams, op.args)
}
paramsBySource, volumeSources, err := volumeParamsBySource(
ctx.environConfig, ctx.config.StorageDir, volumeParams,
)
if err != nil {
return errors.Trace(err)
}
var reschedule []scheduleOp
var volumes []storage.Volume
var volumeAttachments []storage.VolumeAttachment
var statuses []params.EntityStatusArgs
for sourceName, volumeParams := range paramsBySource {
logger.Debugf("creating volumes: %v", volumeParams)
volumeSource := volumeSources[sourceName]
validVolumeParams, validationErrors := validateVolumeParams(volumeSource, volumeParams)
for i, err := range validationErrors {
if err == nil {
continue
}
statuses = append(statuses, params.EntityStatusArgs{
Tag: volumeParams[i].Tag.String(),
Status: params.StatusError,
Info: err.Error(),
})
logger.Debugf(
"failed to validate parameters for %s: %v",
names.ReadableString(volumeParams[i].Tag), err,
)
}
volumeParams = validVolumeParams
if len(volumeParams) == 0 {
continue
}
results, err := volumeSource.CreateVolumes(volumeParams)
if err != nil {
return errors.Annotatef(err, "creating volumes from source %q", sourceName)
}
for i, result := range results {
statuses = append(statuses, params.EntityStatusArgs{
Tag: volumeParams[i].Tag.String(),
Status: params.StatusAttaching,
})
status := &statuses[len(statuses)-1]
if result.Error != nil {
// Reschedule the volume creation.
reschedule = append(reschedule, ops[volumeParams[i].Tag])
// Note: we keep the status as "pending" to indicate
// that we will retry. When we distinguish between
// transient and permanent errors, we will set the
// status to "error" for permanent errors.
status.Status = params.StatusPending
status.Info = result.Error.Error()
logger.Debugf(
"failed to create %s: %v",
names.ReadableString(volumeParams[i].Tag),
result.Error,
)
continue
}
volumes = append(volumes, *result.Volume)
if result.VolumeAttachment != nil {
status.Status = params.StatusAttached
volumeAttachments = append(volumeAttachments, *result.VolumeAttachment)
}
}
}
scheduleOperations(ctx, reschedule...)
setStatus(ctx, statuses)
if len(volumes) == 0 {
return nil
}
// TODO(axw) we need to be able to list volumes in the provider,
// by environment, so that we can "harvest" them if they're
// unknown. This will take care of killing volumes that we fail
// to record in state.
errorResults, err := ctx.config.Volumes.SetVolumeInfo(volumesFromStorage(volumes))
if err != nil {
return errors.Annotate(err, "publishing volumes to state")
}
for i, result := range errorResults {
if result.Error != nil {
logger.Errorf(
"publishing volume %s to state: %v",
volumes[i].Tag.Id(),
result.Error,
)
}
}
for _, v := range volumes {
updateVolume(ctx, v)
}
// Note: the storage provisioner that creates a volume is also
// responsible for creating the volume attachment. It is therefore
// safe to set the volume attachment info after the volume info,
// without leading to the possibility of concurrent, duplicate
// attachments.
err = setVolumeAttachmentInfo(ctx, volumeAttachments)
if err != nil {
return errors.Trace(err)
}
return nil
}
// validateFilesystemMountPoints validates the mount points of filesystems
// being attached to the specified machine. If there are any mount point
// path conflicts, an error will be returned.
func validateFilesystemMountPoints(m *Machine, newFilesystems []filesystemAttachmentTemplate) error {
attachments, err := m.st.MachineFilesystemAttachments(m.MachineTag())
if err != nil {
return errors.Trace(err)
}
existing := make(map[names.FilesystemTag]string)
for _, a := range attachments {
params, ok := a.Params()
if ok {
existing[a.Filesystem()] = params.Location
continue
}
info, err := a.Info()
if err != nil {
return errors.Trace(err)
}
existing[a.Filesystem()] = info.MountPoint
}
storageName := func(
filesystemTag names.FilesystemTag,
storageTag names.StorageTag,
) string {
if storageTag == (names.StorageTag{}) {
return names.ReadableString(filesystemTag)
}
// We know the tag is valid, so ignore the error.
storageName, _ := names.StorageName(storageTag.Id())
return fmt.Sprintf("%q storage", storageName)
}
containsPath := func(a, b string) bool {
a = path.Clean(a) + "/"
b = path.Clean(b) + "/"
return strings.HasPrefix(b, a)
}
// These sets are expected to be small, so sorting and comparing
// adjacent values is not worth the cost of creating a reverse
// lookup from location to filesystem.
for _, template := range newFilesystems {
newMountPoint := template.params.Location
for oldFilesystemTag, oldMountPoint := range existing {
var conflicted, swapOrder bool
if containsPath(oldMountPoint, newMountPoint) {
conflicted = true
} else if containsPath(newMountPoint, oldMountPoint) {
conflicted = true
swapOrder = true
}
if !conflicted {
continue
}
// Get a helpful identifier for the new filesystem. If it
// is being created for a storage instance, then use
// the storage name; otherwise use the filesystem name.
newStorageName := storageName(template.tag, template.storage)
// Likewise for the old filesystem, but this time we'll
// need to consult state.
oldFilesystem, err := m.st.Filesystem(oldFilesystemTag)
if err != nil {
return errors.Trace(err)
}
storageTag, err := oldFilesystem.Storage()
if errors.IsNotAssigned(err) {
storageTag = names.StorageTag{}
} else if err != nil {
return errors.Trace(err)
}
oldStorageName := storageName(oldFilesystemTag, storageTag)
lhs := fmt.Sprintf("mount point %q for %s", oldMountPoint, oldStorageName)
rhs := fmt.Sprintf("mount point %q for %s", newMountPoint, newStorageName)
if swapOrder {
lhs, rhs = rhs, lhs
}
return errors.Errorf("%s contains %s", lhs, rhs)
}
}
return nil
}
// attachFilesystems creates filesystem attachments with the specified parameters.
func attachFilesystems(ctx *context, ops map[params.MachineStorageId]*attachFilesystemOp) error {
filesystemAttachmentParams := make([]storage.FilesystemAttachmentParams, 0, len(ops))
for _, op := range ops {
args := op.args
if args.Path == "" {
args.Path = filepath.Join(ctx.config.StorageDir, args.Filesystem.Id())
}
filesystemAttachmentParams = append(filesystemAttachmentParams, args)
}
paramsBySource, filesystemSources, err := filesystemAttachmentParamsBySource(
ctx.modelConfig,
ctx.config.StorageDir,
filesystemAttachmentParams,
ctx.filesystems,
ctx.managedFilesystemSource,
)
if err != nil {
return errors.Trace(err)
}
var reschedule []scheduleOp
var filesystemAttachments []storage.FilesystemAttachment
var statuses []params.EntityStatusArgs
for sourceName, filesystemAttachmentParams := range paramsBySource {
logger.Debugf("attaching filesystems: %+v", filesystemAttachmentParams)
filesystemSource := filesystemSources[sourceName]
results, err := filesystemSource.AttachFilesystems(filesystemAttachmentParams)
if err != nil {
return errors.Annotatef(err, "attaching filesystems from source %q", sourceName)
}
for i, result := range results {
p := filesystemAttachmentParams[i]
statuses = append(statuses, params.EntityStatusArgs{
Tag: p.Filesystem.String(),
Status: status.StatusAttached,
})
entityStatus := &statuses[len(statuses)-1]
if result.Error != nil {
// Reschedule the filesystem attachment.
id := params.MachineStorageId{
MachineTag: p.Machine.String(),
AttachmentTag: p.Filesystem.String(),
}
reschedule = append(reschedule, ops[id])
// Note: we keep the status as "attaching" to
// indicate that we will retry. When we distinguish
// between transient and permanent errors, we will
// set the status to "error" for permanent errors.
entityStatus.Status = status.StatusAttaching
entityStatus.Info = result.Error.Error()
logger.Debugf(
"failed to attach %s to %s: %v",
names.ReadableString(p.Filesystem),
names.ReadableString(p.Machine),
result.Error,
)
continue
}
filesystemAttachments = append(filesystemAttachments, *result.FilesystemAttachment)
}
}
scheduleOperations(ctx, reschedule...)
setStatus(ctx, statuses)
if err := setFilesystemAttachmentInfo(ctx, filesystemAttachments); err != nil {
return errors.Trace(err)
}
return nil
}
// destroyFilesystems destroys filesystems with the specified parameters.
func destroyFilesystems(ctx *context, ops map[names.FilesystemTag]*destroyFilesystemOp) error {
tags := make([]names.FilesystemTag, 0, len(ops))
for tag := range ops {
tags = append(tags, tag)
}
filesystemParams, err := filesystemParams(ctx, tags)
if err != nil {
return errors.Trace(err)
}
paramsBySource, filesystemSources, err := filesystemParamsBySource(
ctx.modelConfig, ctx.config.StorageDir,
filesystemParams, ctx.managedFilesystemSource,
)
if err != nil {
return errors.Trace(err)
}
var remove []names.Tag
var reschedule []scheduleOp
var statuses []params.EntityStatusArgs
for sourceName, filesystemParams := range paramsBySource {
logger.Debugf("destroying filesystems from %q: %v", sourceName, filesystemParams)
filesystemSource := filesystemSources[sourceName]
validFilesystemParams, validationErrors := validateFilesystemParams(filesystemSource, filesystemParams)
for i, err := range validationErrors {
if err == nil {
continue
}
statuses = append(statuses, params.EntityStatusArgs{
Tag: filesystemParams[i].Tag.String(),
Status: status.StatusError,
Info: err.Error(),
})
logger.Debugf(
"failed to validate parameters for %s: %v",
names.ReadableString(filesystemParams[i].Tag), err,
)
}
filesystemParams = validFilesystemParams
if len(filesystemParams) == 0 {
continue
}
filesystemIds := make([]string, len(filesystemParams))
for i, filesystemParams := range filesystemParams {
filesystem, ok := ctx.filesystems[filesystemParams.Tag]
if !ok {
return errors.NotFoundf("filesystem %s", filesystemParams.Tag.Id())
}
filesystemIds[i] = filesystem.FilesystemId
}
errs, err := filesystemSource.DestroyFilesystems(filesystemIds)
if err != nil {
return errors.Trace(err)
}
for i, err := range errs {
tag := filesystemParams[i].Tag
if err == nil {
remove = append(remove, tag)
continue
}
// Failed to destroy filesystem; reschedule and update status.
reschedule = append(reschedule, ops[tag])
statuses = append(statuses, params.EntityStatusArgs{
Tag: tag.String(),
Status: status.StatusDestroying,
Info: err.Error(),
})
}
}
scheduleOperations(ctx, reschedule...)
setStatus(ctx, statuses)
if err := removeEntities(ctx, remove); err != nil {
return errors.Annotate(err, "removing filesystems from state")
}
return nil
}