// nextSubnetIPAddress returns the next available IP address in the given subnet.
func nextSubnetIPAddress(
nicClient network.InterfacesClient,
resourceGroup string,
subnet *network.Subnet,
) (string, error) {
_, ipnet, err := net.ParseCIDR(to.String(subnet.Properties.AddressPrefix))
if err != nil {
return "", errors.Annotate(err, "parsing subnet prefix")
}
results, err := nicClient.List(resourceGroup)
if err != nil {
return "", errors.Annotate(err, "listing NICs")
}
// Azure reserves the first 4 addresses in the subnet.
var ipsInUse []net.IP
if results.Value != nil {
ipsInUse = make([]net.IP, 0, len(*results.Value))
for _, item := range *results.Value {
if item.Properties.IPConfigurations == nil {
continue
}
for _, ipConfiguration := range *item.Properties.IPConfigurations {
if to.String(ipConfiguration.Properties.Subnet.ID) != to.String(subnet.ID) {
continue
}
ip := net.ParseIP(to.String(ipConfiguration.Properties.PrivateIPAddress))
if ip != nil {
ipsInUse = append(ipsInUse, ip)
}
}
}
}
ip, err := iputils.NextSubnetIP(ipnet, ipsInUse)
if err != nil {
return "", errors.Trace(err)
}
return ip.String(), nil
}
// instanceNetworkInterfaces lists all network interfaces in the resource
// group, and returns a mapping from instance ID to the network interfaces
// associated with that instance.
func instanceNetworkInterfaces(
callAPI callAPIFunc,
resourceGroup string,
nicClient network.InterfacesClient,
) (map[instance.Id][]network.Interface, error) {
var nicsResult network.InterfaceListResult
if err := callAPI(func() (autorest.Response, error) {
var err error
nicsResult, err = nicClient.List(resourceGroup)
return nicsResult.Response, err
}); err != nil {
return nil, errors.Annotate(err, "listing network interfaces")
}
if nicsResult.Value == nil || len(*nicsResult.Value) == 0 {
return nil, nil
}
instanceNics := make(map[instance.Id][]network.Interface)
for _, nic := range *nicsResult.Value {
instanceId := instance.Id(toTags(nic.Tags)[jujuMachineNameTag])
instanceNics[instanceId] = append(instanceNics[instanceId], nic)
}
return instanceNics, nil
}
// allInstances returns all of the instances in the given resource group,
// and optionally ensures that each instance's addresses are up-to-date.
func (env *azureEnviron) allInstances(
resourceGroup string,
refreshAddresses bool,
) ([]instance.Instance, error) {
env.mu.Lock()
vmClient := compute.VirtualMachinesClient{env.compute}
nicClient := network.InterfacesClient{env.network}
pipClient := network.PublicIPAddressesClient{env.network}
env.mu.Unlock()
// Due to how deleting instances works, we have to get creative about
// listing instances. We list NICs and return an instance for each
// unique value of the jujuMachineNameTag tag.
//
// The machine provisioner will call AllInstances so it can delete
// unknown instances. StopInstances must delete VMs before NICs and
// public IPs, because a VM cannot have less than 1 NIC. Thus, we can
// potentially delete a VM but then fail to delete its NIC.
nicsResult, err := nicClient.List(resourceGroup)
if err != nil {
if nicsResult.Response.Response != nil && nicsResult.StatusCode == http.StatusNotFound {
// This will occur if the resource group does not
// exist, e.g. in a fresh hosted environment.
return nil, nil
}
return nil, errors.Trace(err)
}
if nicsResult.Value == nil || len(*nicsResult.Value) == 0 {
return nil, nil
}
// Create an azureInstance for each VM.
result, err := vmClient.List(resourceGroup)
if err != nil {
return nil, errors.Annotate(err, "listing virtual machines")
}
vmNames := make(set.Strings)
var azureInstances []*azureInstance
if result.Value != nil {
azureInstances = make([]*azureInstance, len(*result.Value))
for i, vm := range *result.Value {
inst := &azureInstance{vm, env, nil, nil}
azureInstances[i] = inst
vmNames.Add(to.String(vm.Name))
}
}
// Create additional azureInstances for NICs without machines. See
// comments above for rationale. This needs to happen before calling
// setInstanceAddresses, so we still associate the NICs/PIPs.
for _, nic := range *nicsResult.Value {
vmName, ok := toTags(nic.Tags)[jujuMachineNameTag]
if !ok || vmNames.Contains(vmName) {
continue
}
vm := compute.VirtualMachine{
Name: to.StringPtr(vmName),
Properties: &compute.VirtualMachineProperties{
ProvisioningState: to.StringPtr("Partially Deleted"),
},
}
inst := &azureInstance{vm, env, nil, nil}
azureInstances = append(azureInstances, inst)
vmNames.Add(to.String(vm.Name))
}
if len(azureInstances) > 0 && refreshAddresses {
if err := setInstanceAddresses(
pipClient, resourceGroup, azureInstances, nicsResult,
); err != nil {
return nil, errors.Trace(err)
}
}
instances := make([]instance.Instance, len(azureInstances))
for i, inst := range azureInstances {
instances[i] = inst
}
return instances, nil
}
// deleteInstances deletes a virtual machine and all of the resources that
// it owns, and any corresponding network security rules.
func deleteInstance(
inst *azureInstance,
computeClient compute.ManagementClient,
networkClient network.ManagementClient,
storageClient internalazurestorage.Client,
) error {
vmName := string(inst.Id())
vmClient := compute.VirtualMachinesClient{computeClient}
nicClient := network.InterfacesClient{networkClient}
nsgClient := network.SecurityGroupsClient{networkClient}
securityRuleClient := network.SecurityRulesClient{networkClient}
publicIPClient := network.PublicIPAddressesClient{networkClient}
logger.Debugf("deleting instance %q", vmName)
logger.Debugf("- deleting virtual machine")
deleteResult, err := vmClient.Delete(inst.env.resourceGroup, vmName)
if err != nil {
if deleteResult.Response == nil || deleteResult.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting virtual machine")
}
}
// Delete the VM's OS disk VHD.
logger.Debugf("- deleting OS VHD")
blobClient := storageClient.GetBlobService()
if _, err := blobClient.DeleteBlobIfExists(osDiskVHDContainer, vmName); err != nil {
return errors.Annotate(err, "deleting OS VHD")
}
// Delete network security rules that refer to the VM.
logger.Debugf("- deleting security rules")
if err := deleteInstanceNetworkSecurityRules(
inst.env.resourceGroup, inst.Id(), nsgClient, securityRuleClient,
); err != nil {
return errors.Annotate(err, "deleting network security rules")
}
// Detach public IPs from NICs. This must be done before public
// IPs can be deleted. In the future, VMs may not necessarily
// have a public IP, so we don't use the presence of a public
// IP to indicate the existence of an instance.
logger.Debugf("- detaching public IP addresses")
for _, nic := range inst.networkInterfaces {
if nic.Properties.IPConfigurations == nil {
continue
}
var detached bool
for i, ipConfiguration := range *nic.Properties.IPConfigurations {
if ipConfiguration.Properties.PublicIPAddress == nil {
continue
}
ipConfiguration.Properties.PublicIPAddress = nil
(*nic.Properties.IPConfigurations)[i] = ipConfiguration
detached = true
}
if detached {
if _, err := nicClient.CreateOrUpdate(
inst.env.resourceGroup, to.String(nic.Name), nic,
); err != nil {
return errors.Annotate(err, "detaching public IP addresses")
}
}
}
// Delete public IPs.
logger.Debugf("- deleting public IPs")
for _, pip := range inst.publicIPAddresses {
pipName := to.String(pip.Name)
logger.Tracef("deleting public IP %q", pipName)
result, err := publicIPClient.Delete(inst.env.resourceGroup, pipName)
if err != nil {
if result.Response == nil || result.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting public IP")
}
}
}
// Delete NICs.
//
// NOTE(axw) this *must* be deleted last, or we risk leaking resources.
logger.Debugf("- deleting network interfaces")
for _, nic := range inst.networkInterfaces {
nicName := to.String(nic.Name)
logger.Tracef("deleting NIC %q", nicName)
result, err := nicClient.Delete(inst.env.resourceGroup, nicName)
if err != nil {
if result.Response == nil || result.StatusCode != http.StatusNotFound {
return errors.Annotate(err, "deleting NIC")
}
}
}
return nil
}
func newNetworkProfile(
client network.ManagementClient,
vmName string,
apiPort *int,
internalSubnet *network.Subnet,
resourceGroup string,
location string,
tags map[string]string,
) (*compute.NetworkProfile, error) {
logger.Debugf("creating network profile for %q", vmName)
// Create a public IP for the NIC. Public IP addresses are dynamic.
logger.Debugf("- allocating public IP address")
pipClient := network.PublicIPAddressesClient{client}
publicIPAddressParams := network.PublicIPAddress{
Location: to.StringPtr(location),
Tags: toTagsPtr(tags),
Properties: &network.PublicIPAddressPropertiesFormat{
PublicIPAllocationMethod: network.Dynamic,
},
}
publicIPAddressName := vmName + "-public-ip"
publicIPAddress, err := pipClient.CreateOrUpdate(resourceGroup, publicIPAddressName, publicIPAddressParams)
if err != nil {
return nil, errors.Annotatef(err, "creating public IP address for %q", vmName)
}
// Determine the next available private IP address.
nicClient := network.InterfacesClient{client}
privateIPAddress, err := nextSubnetIPAddress(nicClient, resourceGroup, internalSubnet)
if err != nil {
return nil, errors.Annotatef(err, "querying private IP addresses")
}
// Create a primary NIC for the machine. This needs to be static, so
// that we can create security rules that don't become invalid.
logger.Debugf("- creating primary NIC")
ipConfigurations := []network.InterfaceIPConfiguration{{
Name: to.StringPtr("primary"),
Properties: &network.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr(privateIPAddress),
PrivateIPAllocationMethod: network.Static,
Subnet: &network.SubResource{internalSubnet.ID},
PublicIPAddress: &network.SubResource{publicIPAddress.ID},
},
}}
primaryNicName := vmName + "-primary"
primaryNicParams := network.Interface{
Location: to.StringPtr(location),
Tags: toTagsPtr(tags),
Properties: &network.InterfacePropertiesFormat{
IPConfigurations: &ipConfigurations,
},
}
primaryNic, err := nicClient.CreateOrUpdate(resourceGroup, primaryNicName, primaryNicParams)
if err != nil {
return nil, errors.Annotatef(err, "creating network interface for %q", vmName)
}
// Create a network security rule for the machine if we need to open
// the API server port.
if apiPort != nil {
logger.Debugf("- querying network security group")
securityGroupClient := network.SecurityGroupsClient{client}
securityGroupName := internalSecurityGroupName
securityGroup, err := securityGroupClient.Get(resourceGroup, securityGroupName)
if err != nil {
return nil, errors.Annotate(err, "querying network security group")
}
// NOTE(axw) this looks like TOCTTOU race territory, but it's
// safe because we only allocate/deallocate rules in this
// range during machine (de)provisioning, which is managed by
// a single goroutine. Non-internal ports are managed by the
// firewaller exclusively.
nextPriority, err := nextSecurityRulePriority(
securityGroup,
securityRuleInternalSSHInbound+1,
securityRuleInternalMax,
)
if err != nil {
return nil, errors.Trace(err)
}
apiSecurityRuleName := fmt.Sprintf("%s-api", vmName)
apiSecurityRule := network.SecurityRule{
Name: to.StringPtr(apiSecurityRuleName),
Properties: &network.SecurityRulePropertiesFormat{
Description: to.StringPtr("Allow API access to server machines"),
Protocol: network.SecurityRuleProtocolTCP,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr(privateIPAddress),
DestinationPortRange: to.StringPtr(fmt.Sprint(*apiPort)),
Access: network.Allow,
Priority: to.IntPtr(nextPriority),
Direction: network.Inbound,
},
}
logger.Debugf("- creating API network security rule")
securityRuleClient := network.SecurityRulesClient{client}
_, err = securityRuleClient.CreateOrUpdate(
resourceGroup, securityGroupName, apiSecurityRuleName, apiSecurityRule,
)
if err != nil {
return nil, errors.Annotate(err, "creating API network security rule")
}
}
// For now we only attach a single, flat network to each machine.
networkInterfaces := []compute.NetworkInterfaceReference{{
ID: primaryNic.ID,
Properties: &compute.NetworkInterfaceReferenceProperties{
Primary: to.BoolPtr(true),
},
}}
return &compute.NetworkProfile{&networkInterfaces}, nil
}