// interfacePublicIPAddresses lists all public IP addresses in the resource
// group, and returns a mapping from instance ID to the public IP addresses
// associated with that instance.
func instancePublicIPAddresses(
callAPI callAPIFunc,
resourceGroup string,
pipClient network.PublicIPAddressesClient,
) (map[instance.Id][]network.PublicIPAddress, error) {
var pipsResult network.PublicIPAddressListResult
if err := callAPI(func() (autorest.Response, error) {
var err error
pipsResult, err = pipClient.List(resourceGroup)
return pipsResult.Response, err
}); err != nil {
return nil, errors.Annotate(err, "listing public IP addresses")
}
if pipsResult.Value == nil || len(*pipsResult.Value) == 0 {
return nil, nil
}
instancePips := make(map[instance.Id][]network.PublicIPAddress)
for _, pip := range *pipsResult.Value {
instanceId := instance.Id(toTags(pip.Tags)[jujuMachineNameTag])
instancePips[instanceId] = append(instancePips[instanceId], pip)
}
return instancePips, nil
}
// deleteInstances deletes a virtual machine and all of the resources that
// it owns, and any corresponding network security rules.
func deleteInstance(
inst *azureInstance,
computeClient compute.ManagementClient,
networkClient network.ManagementClient,
storageClient internalazurestorage.Client,
) error {
vmName := string(inst.Id())
vmClient := compute.VirtualMachinesClient{computeClient}
nicClient := network.InterfacesClient{networkClient}
nsgClient := network.SecurityGroupsClient{networkClient}
securityRuleClient := network.SecurityRulesClient{networkClient}
publicIPClient := network.PublicIPAddressesClient{networkClient}
logger.Debugf("deleting instance %q", vmName)
logger.Debugf("- deleting virtual machine")
deleteResult, err := vmClient.Delete(inst.env.resourceGroup, vmName)
if err != nil {
if deleteResult.Response == nil || deleteResult.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting virtual machine")
}
}
// Delete the VM's OS disk VHD.
logger.Debugf("- deleting OS VHD")
blobClient := storageClient.GetBlobService()
if _, err := blobClient.DeleteBlobIfExists(osDiskVHDContainer, vmName); err != nil {
return errors.Annotate(err, "deleting OS VHD")
}
// Delete network security rules that refer to the VM.
logger.Debugf("- deleting security rules")
if err := deleteInstanceNetworkSecurityRules(
inst.env.resourceGroup, inst.Id(), nsgClient, securityRuleClient,
); err != nil {
return errors.Annotate(err, "deleting network security rules")
}
// Detach public IPs from NICs. This must be done before public
// IPs can be deleted. In the future, VMs may not necessarily
// have a public IP, so we don't use the presence of a public
// IP to indicate the existence of an instance.
logger.Debugf("- detaching public IP addresses")
for _, nic := range inst.networkInterfaces {
if nic.Properties.IPConfigurations == nil {
continue
}
var detached bool
for i, ipConfiguration := range *nic.Properties.IPConfigurations {
if ipConfiguration.Properties.PublicIPAddress == nil {
continue
}
ipConfiguration.Properties.PublicIPAddress = nil
(*nic.Properties.IPConfigurations)[i] = ipConfiguration
detached = true
}
if detached {
if _, err := nicClient.CreateOrUpdate(
inst.env.resourceGroup, to.String(nic.Name), nic,
); err != nil {
return errors.Annotate(err, "detaching public IP addresses")
}
}
}
// Delete public IPs.
logger.Debugf("- deleting public IPs")
for _, pip := range inst.publicIPAddresses {
pipName := to.String(pip.Name)
logger.Tracef("deleting public IP %q", pipName)
result, err := publicIPClient.Delete(inst.env.resourceGroup, pipName)
if err != nil {
if result.Response == nil || result.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting public IP")
}
}
}
// Delete NICs.
//
// NOTE(axw) this *must* be deleted last, or we risk leaking resources.
logger.Debugf("- deleting network interfaces")
for _, nic := range inst.networkInterfaces {
nicName := to.String(nic.Name)
logger.Tracef("deleting NIC %q", nicName)
result, err := nicClient.Delete(inst.env.resourceGroup, nicName)
if err != nil {
if result.Response == nil || result.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting NIC")
}
}
}
return nil
}
func newNetworkProfile(
client network.ManagementClient,
vmName string,
apiPort *int,
internalSubnet *network.Subnet,
resourceGroup string,
location string,
tags map[string]string,
) (*compute.NetworkProfile, error) {
logger.Debugf("creating network profile for %q", vmName)
// Create a public IP for the NIC. Public IP addresses are dynamic.
logger.Debugf("- allocating public IP address")
pipClient := network.PublicIPAddressesClient{client}
publicIPAddressParams := network.PublicIPAddress{
Location: to.StringPtr(location),
Tags: toTagsPtr(tags),
Properties: &network.PublicIPAddressPropertiesFormat{
PublicIPAllocationMethod: network.Dynamic,
},
}
publicIPAddressName := vmName + "-public-ip"
publicIPAddress, err := pipClient.CreateOrUpdate(resourceGroup, publicIPAddressName, publicIPAddressParams)
if err != nil {
return nil, errors.Annotatef(err, "creating public IP address for %q", vmName)
}
// Determine the next available private IP address.
nicClient := network.InterfacesClient{client}
privateIPAddress, err := nextSubnetIPAddress(nicClient, resourceGroup, internalSubnet)
if err != nil {
return nil, errors.Annotatef(err, "querying private IP addresses")
}
// Create a primary NIC for the machine. This needs to be static, so
// that we can create security rules that don't become invalid.
logger.Debugf("- creating primary NIC")
ipConfigurations := []network.InterfaceIPConfiguration{{
Name: to.StringPtr("primary"),
Properties: &network.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr(privateIPAddress),
PrivateIPAllocationMethod: network.Static,
Subnet: &network.SubResource{internalSubnet.ID},
PublicIPAddress: &network.SubResource{publicIPAddress.ID},
},
}}
primaryNicName := vmName + "-primary"
primaryNicParams := network.Interface{
Location: to.StringPtr(location),
Tags: toTagsPtr(tags),
Properties: &network.InterfacePropertiesFormat{
IPConfigurations: &ipConfigurations,
},
}
primaryNic, err := nicClient.CreateOrUpdate(resourceGroup, primaryNicName, primaryNicParams)
if err != nil {
return nil, errors.Annotatef(err, "creating network interface for %q", vmName)
}
// Create a network security rule for the machine if we need to open
// the API server port.
if apiPort != nil {
logger.Debugf("- querying network security group")
securityGroupClient := network.SecurityGroupsClient{client}
securityGroupName := internalSecurityGroupName
securityGroup, err := securityGroupClient.Get(resourceGroup, securityGroupName)
if err != nil {
return nil, errors.Annotate(err, "querying network security group")
}
// NOTE(axw) this looks like TOCTTOU race territory, but it's
// safe because we only allocate/deallocate rules in this
// range during machine (de)provisioning, which is managed by
// a single goroutine. Non-internal ports are managed by the
// firewaller exclusively.
nextPriority, err := nextSecurityRulePriority(
securityGroup,
securityRuleInternalSSHInbound+1,
securityRuleInternalMax,
)
if err != nil {
return nil, errors.Trace(err)
}
apiSecurityRuleName := fmt.Sprintf("%s-api", vmName)
apiSecurityRule := network.SecurityRule{
Name: to.StringPtr(apiSecurityRuleName),
Properties: &network.SecurityRulePropertiesFormat{
Description: to.StringPtr("Allow API access to server machines"),
Protocol: network.SecurityRuleProtocolTCP,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr(privateIPAddress),
DestinationPortRange: to.StringPtr(fmt.Sprint(*apiPort)),
Access: network.Allow,
Priority: to.IntPtr(nextPriority),
Direction: network.Inbound,
},
}
logger.Debugf("- creating API network security rule")
securityRuleClient := network.SecurityRulesClient{client}
_, err = securityRuleClient.CreateOrUpdate(
resourceGroup, securityGroupName, apiSecurityRuleName, apiSecurityRule,
)
if err != nil {
return nil, errors.Annotate(err, "creating API network security rule")
}
}
// For now we only attach a single, flat network to each machine.
networkInterfaces := []compute.NetworkInterfaceReference{{
ID: primaryNic.ID,
Properties: &compute.NetworkInterfaceReferenceProperties{
Primary: to.BoolPtr(true),
},
}}
return &compute.NetworkProfile{&networkInterfaces}, nil
}
// setInstanceAddresses queries Azure for the NICs and public IPs associated
// with the given set of instances. This assumes that the instances'
// VirtualMachines are up-to-date, and that there are no concurrent accesses
// to the instances.
func setInstanceAddresses(
pipClient network.PublicIPAddressesClient,
resourceGroup string,
instances []*azureInstance,
nicsResult network.InterfaceListResult,
) (err error) {
instanceNics := make(map[instance.Id][]network.Interface)
instancePips := make(map[instance.Id][]network.PublicIPAddress)
for _, inst := range instances {
instanceNics[inst.Id()] = nil
instancePips[inst.Id()] = nil
}
// When setAddresses returns without error, update each
// instance's network interfaces and public IP addresses.
setInstanceFields := func(inst *azureInstance) {
inst.networkInterfaces = instanceNics[inst.Id()]
inst.publicIPAddresses = instancePips[inst.Id()]
}
defer func() {
if err != nil {
return
}
for _, inst := range instances {
setInstanceFields(inst)
}
}()
// We do not rely on references because of how StopInstances works.
// In order to not leak resources we must not delete the virtual
// machine until after all of its dependencies are deleted.
//
// NICs and PIPs cannot be deleted until they have no references.
// Thus, we cannot delete a PIP until there is no reference to it
// in any NICs, and likewise we cannot delete a NIC until there
// is no reference to it in any virtual machine.
if nicsResult.Value != nil {
for _, nic := range *nicsResult.Value {
instanceId := instance.Id(toTags(nic.Tags)[jujuMachineNameTag])
if _, ok := instanceNics[instanceId]; !ok {
continue
}
instanceNics[instanceId] = append(instanceNics[instanceId], nic)
}
}
pipsResult, err := pipClient.List(resourceGroup)
if err != nil {
return errors.Annotate(err, "listing public IP addresses")
}
if pipsResult.Value != nil {
for _, pip := range *pipsResult.Value {
instanceId := instance.Id(toTags(pip.Tags)[jujuMachineNameTag])
if _, ok := instanceNics[instanceId]; !ok {
continue
}
instancePips[instanceId] = append(instancePips[instanceId], pip)
}
}
// Fields will be assigned to instances by the deferred call.
return nil
}