// updateVirtualMachines updates virtual machines in the given map by iterating
// through the list of instance IDs in order, and updating each corresponding
// virtual machine at most once.
func (v *azureVolumeSource) updateVirtualMachines(
virtualMachines map[instance.Id]*maybeVirtualMachine, instanceIds []instance.Id,
) ([]error, error) {
results := make([]error, len(instanceIds))
vmsClient := compute.VirtualMachinesClient{v.env.compute}
for i, instanceId := range instanceIds {
vm, ok := virtualMachines[instanceId]
if !ok {
continue
}
if vm.err != nil {
results[i] = vm.err
continue
}
if err := v.env.callAPI(func() (autorest.Response, error) {
return vmsClient.CreateOrUpdate(
v.env.resourceGroup, to.String(vm.vm.Name), *vm.vm,
nil, // abort channel
)
}); err != nil {
results[i] = err
vm.err = err
continue
}
// successfully updated, don't update again
delete(virtualMachines, instanceId)
}
return results, nil
}
// virtualMachines returns a mapping of instance IDs to VirtualMachines and
// errors, for each of the specified instance IDs.
func (v *azureVolumeSource) virtualMachines(instanceIds []instance.Id) (map[instance.Id]*maybeVirtualMachine, error) {
vmsClient := compute.VirtualMachinesClient{v.env.compute}
var result compute.VirtualMachineListResult
if err := v.env.callAPI(func() (autorest.Response, error) {
var err error
result, err = vmsClient.List(v.env.resourceGroup)
return result.Response, err
}); err != nil {
return nil, errors.Annotate(err, "listing virtual machines")
}
all := make(map[instance.Id]*compute.VirtualMachine)
if result.Value != nil {
for _, vm := range *result.Value {
vmCopy := vm
all[instance.Id(to.String(vm.Name))] = &vmCopy
}
}
results := make(map[instance.Id]*maybeVirtualMachine)
for _, id := range instanceIds {
result := &maybeVirtualMachine{vm: all[id]}
if result.vm == nil {
result.err = errors.NotFoundf("instance %v", id)
}
results[id] = result
}
return results, nil
}
// allInstances returns all of the instances in the given resource group,
// and optionally ensures that each instance's addresses are up-to-date.
func (env *azureEnviron) allInstances(
resourceGroup string,
refreshAddresses bool,
) ([]instance.Instance, error) {
env.mu.Lock()
vmClient := compute.VirtualMachinesClient{env.compute}
nicClient := network.InterfacesClient{env.network}
pipClient := network.PublicIPAddressesClient{env.network}
env.mu.Unlock()
// Due to how deleting instances works, we have to get creative about
// listing instances. We list NICs and return an instance for each
// unique value of the jujuMachineNameTag tag.
//
// The machine provisioner will call AllInstances so it can delete
// unknown instances. StopInstances must delete VMs before NICs and
// public IPs, because a VM cannot have less than 1 NIC. Thus, we can
// potentially delete a VM but then fail to delete its NIC.
nicsResult, err := nicClient.List(resourceGroup)
if err != nil {
if nicsResult.Response.Response != nil && nicsResult.StatusCode == http.StatusNotFound {
// This will occur if the resource group does not
// exist, e.g. in a fresh hosted environment.
return nil, nil
}
return nil, errors.Trace(err)
}
if nicsResult.Value == nil || len(*nicsResult.Value) == 0 {
return nil, nil
}
// Create an azureInstance for each VM.
result, err := vmClient.List(resourceGroup)
if err != nil {
return nil, errors.Annotate(err, "listing virtual machines")
}
vmNames := make(set.Strings)
var azureInstances []*azureInstance
if result.Value != nil {
azureInstances = make([]*azureInstance, len(*result.Value))
for i, vm := range *result.Value {
inst := &azureInstance{vm, env, nil, nil}
azureInstances[i] = inst
vmNames.Add(to.String(vm.Name))
}
}
// Create additional azureInstances for NICs without machines. See
// comments above for rationale. This needs to happen before calling
// setInstanceAddresses, so we still associate the NICs/PIPs.
for _, nic := range *nicsResult.Value {
vmName, ok := toTags(nic.Tags)[jujuMachineNameTag]
if !ok || vmNames.Contains(vmName) {
continue
}
vm := compute.VirtualMachine{
Name: to.StringPtr(vmName),
Properties: &compute.VirtualMachineProperties{
ProvisioningState: to.StringPtr("Partially Deleted"),
},
}
inst := &azureInstance{vm, env, nil, nil}
azureInstances = append(azureInstances, inst)
vmNames.Add(to.String(vm.Name))
}
if len(azureInstances) > 0 && refreshAddresses {
if err := setInstanceAddresses(
pipClient, resourceGroup, azureInstances, nicsResult,
); err != nil {
return nil, errors.Trace(err)
}
}
instances := make([]instance.Instance, len(azureInstances))
for i, inst := range azureInstances {
instances[i] = inst
}
return instances, nil
}
// deleteInstances deletes a virtual machine and all of the resources that
// it owns, and any corresponding network security rules.
func deleteInstance(
inst *azureInstance,
computeClient compute.ManagementClient,
networkClient network.ManagementClient,
storageClient internalazurestorage.Client,
) error {
vmName := string(inst.Id())
vmClient := compute.VirtualMachinesClient{computeClient}
nicClient := network.InterfacesClient{networkClient}
nsgClient := network.SecurityGroupsClient{networkClient}
securityRuleClient := network.SecurityRulesClient{networkClient}
publicIPClient := network.PublicIPAddressesClient{networkClient}
logger.Debugf("deleting instance %q", vmName)
logger.Debugf("- deleting virtual machine")
deleteResult, err := vmClient.Delete(inst.env.resourceGroup, vmName)
if err != nil {
if deleteResult.Response == nil || deleteResult.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting virtual machine")
}
}
// Delete the VM's OS disk VHD.
logger.Debugf("- deleting OS VHD")
blobClient := storageClient.GetBlobService()
if _, err := blobClient.DeleteBlobIfExists(osDiskVHDContainer, vmName); err != nil {
return errors.Annotate(err, "deleting OS VHD")
}
// Delete network security rules that refer to the VM.
logger.Debugf("- deleting security rules")
if err := deleteInstanceNetworkSecurityRules(
inst.env.resourceGroup, inst.Id(), nsgClient, securityRuleClient,
); err != nil {
return errors.Annotate(err, "deleting network security rules")
}
// Detach public IPs from NICs. This must be done before public
// IPs can be deleted. In the future, VMs may not necessarily
// have a public IP, so we don't use the presence of a public
// IP to indicate the existence of an instance.
logger.Debugf("- detaching public IP addresses")
for _, nic := range inst.networkInterfaces {
if nic.Properties.IPConfigurations == nil {
continue
}
var detached bool
for i, ipConfiguration := range *nic.Properties.IPConfigurations {
if ipConfiguration.Properties.PublicIPAddress == nil {
continue
}
ipConfiguration.Properties.PublicIPAddress = nil
(*nic.Properties.IPConfigurations)[i] = ipConfiguration
detached = true
}
if detached {
if _, err := nicClient.CreateOrUpdate(
inst.env.resourceGroup, to.String(nic.Name), nic,
); err != nil {
return errors.Annotate(err, "detaching public IP addresses")
}
}
}
// Delete public IPs.
logger.Debugf("- deleting public IPs")
for _, pip := range inst.publicIPAddresses {
pipName := to.String(pip.Name)
logger.Tracef("deleting public IP %q", pipName)
result, err := publicIPClient.Delete(inst.env.resourceGroup, pipName)
if err != nil {
if result.Response == nil || result.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting public IP")
}
}
}
// Delete NICs.
//
// NOTE(axw) this *must* be deleted last, or we risk leaking resources.
logger.Debugf("- deleting network interfaces")
for _, nic := range inst.networkInterfaces {
nicName := to.String(nic.Name)
logger.Tracef("deleting NIC %q", nicName)
result, err := nicClient.Delete(inst.env.resourceGroup, nicName)
if err != nil {
if result.Response == nil || result.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting NIC")
}
}
}
return nil
}
// createVirtualMachine creates a virtual machine and related resources.
//
// All resources created are tagged with the specified "vmTags", so if
// this function fails then all resources can be deleted by tag.
func createVirtualMachine(
resourceGroup, location, vmName string,
vmTags, envTags map[string]string,
instanceSpec *instances.InstanceSpec,
instanceConfig *instancecfg.InstanceConfig,
distributionGroupFunc func() ([]instance.Id, error),
instancesFunc func([]instance.Id) ([]instance.Instance, error),
apiPort *int,
internalNetworkSubnet *network.Subnet,
nsgID, storageEndpoint, storageAccountName string,
networkClient network.ManagementClient,
vmClient compute.VirtualMachinesClient,
availabilitySetClient compute.AvailabilitySetsClient,
vmExtensionClient compute.VirtualMachineExtensionsClient,
) (compute.VirtualMachine, error) {
storageProfile, err := newStorageProfile(
vmName, instanceConfig.Series,
instanceSpec, storageEndpoint, storageAccountName,
)
if err != nil {
return compute.VirtualMachine{}, errors.Annotate(err, "creating storage profile")
}
osProfile, seriesOS, err := newOSProfile(vmName, instanceConfig)
if err != nil {
return compute.VirtualMachine{}, errors.Annotate(err, "creating OS profile")
}
networkProfile, err := newNetworkProfile(
networkClient, vmName, apiPort,
internalNetworkSubnet, nsgID,
resourceGroup, location, vmTags,
)
if err != nil {
return compute.VirtualMachine{}, errors.Annotate(err, "creating network profile")
}
availabilitySetId, err := createAvailabilitySet(
availabilitySetClient,
vmName, resourceGroup, location,
vmTags, envTags,
distributionGroupFunc, instancesFunc,
)
if err != nil {
return compute.VirtualMachine{}, errors.Annotate(err, "creating availability set")
}
vmArgs := compute.VirtualMachine{
Location: to.StringPtr(location),
Tags: toTagsPtr(vmTags),
Properties: &compute.VirtualMachineProperties{
HardwareProfile: &compute.HardwareProfile{
VMSize: compute.VirtualMachineSizeTypes(
instanceSpec.InstanceType.Name,
),
},
StorageProfile: storageProfile,
OsProfile: osProfile,
NetworkProfile: networkProfile,
AvailabilitySet: &compute.SubResource{
ID: to.StringPtr(availabilitySetId),
},
},
}
vm, err := vmClient.CreateOrUpdate(resourceGroup, vmName, vmArgs)
if err != nil {
return compute.VirtualMachine{}, errors.Annotate(err, "creating virtual machine")
}
// On Windows and CentOS, we must add the CustomScript VM
// extension to run the CustomData script.
switch seriesOS {
case os.Windows, os.CentOS:
if err := createVMExtension(
vmExtensionClient, seriesOS,
resourceGroup, vmName, location, vmTags,
); err != nil {
return compute.VirtualMachine{}, errors.Annotate(
err, "creating virtual machine extension",
)
}
}
return vm, nil
}