// deleteInstanceNetworkSecurityRules deletes network security rules in the
// internal network security group that correspond to the specified machine.
//
// This is expected to delete *all* security rules related to the instance,
// i.e. both the ones opened by OpenPorts above, and the ones opened for API
// access.
func deleteInstanceNetworkSecurityRules(
resourceGroup string, id instance.Id,
nsgClient network.SecurityGroupsClient,
securityRuleClient network.SecurityRulesClient,
) error {
nsg, err := nsgClient.Get(resourceGroup, internalSecurityGroupName)
if err != nil {
return errors.Annotate(err, "querying network security group")
}
if nsg.Properties.SecurityRules == nil {
return nil
}
prefix := instanceNetworkSecurityRulePrefix(id)
for _, rule := range *nsg.Properties.SecurityRules {
ruleName := to.String(rule.Name)
if !strings.HasPrefix(ruleName, prefix) {
continue
}
result, err := securityRuleClient.Delete(
resourceGroup,
internalSecurityGroupName,
ruleName,
)
if err != nil {
if result.Response == nil || result.StatusCode != http.StatusNotFound {
return errors.Annotatef(err, "deleting security rule %q", ruleName)
}
}
}
return nil
}
// deleteInstanceNetworkSecurityRules deletes network security rules in the
// internal network security group that correspond to the specified machine.
//
// This is expected to delete *all* security rules related to the instance,
// i.e. both the ones opened by OpenPorts above, and the ones opened for API
// access.
func deleteInstanceNetworkSecurityRules(
resourceGroup string, id instance.Id,
nsgClient network.SecurityGroupsClient,
securityRuleClient network.SecurityRulesClient,
callAPI callAPIFunc,
) error {
var nsg network.SecurityGroup
if err := callAPI(func() (autorest.Response, error) {
var err error
nsg, err = nsgClient.Get(resourceGroup, internalSecurityGroupName, "")
return nsg.Response, err
}); err != nil {
return errors.Annotate(err, "querying network security group")
}
if nsg.Properties.SecurityRules == nil {
return nil
}
prefix := instanceNetworkSecurityRulePrefix(id)
for _, rule := range *nsg.Properties.SecurityRules {
ruleName := to.String(rule.Name)
if !strings.HasPrefix(ruleName, prefix) {
continue
}
var result autorest.Response
err := callAPI(func() (autorest.Response, error) {
var err error
result, err = securityRuleClient.Delete(
resourceGroup,
internalSecurityGroupName,
ruleName,
nil, // abort channel
)
return result, err
})
if err != nil {
if result.Response == nil || result.StatusCode != http.StatusNotFound {
return errors.Annotatef(err, "deleting security rule %q", ruleName)
}
}
}
return nil
}
func newNetworkProfile(
client network.ManagementClient,
vmName string,
apiPort *int,
internalSubnet *network.Subnet,
resourceGroup string,
location string,
tags map[string]string,
) (*compute.NetworkProfile, error) {
logger.Debugf("creating network profile for %q", vmName)
// Create a public IP for the NIC. Public IP addresses are dynamic.
logger.Debugf("- allocating public IP address")
pipClient := network.PublicIPAddressesClient{client}
publicIPAddressParams := network.PublicIPAddress{
Location: to.StringPtr(location),
Tags: toTagsPtr(tags),
Properties: &network.PublicIPAddressPropertiesFormat{
PublicIPAllocationMethod: network.Dynamic,
},
}
publicIPAddressName := vmName + "-public-ip"
publicIPAddress, err := pipClient.CreateOrUpdate(resourceGroup, publicIPAddressName, publicIPAddressParams)
if err != nil {
return nil, errors.Annotatef(err, "creating public IP address for %q", vmName)
}
// Determine the next available private IP address.
nicClient := network.InterfacesClient{client}
privateIPAddress, err := nextSubnetIPAddress(nicClient, resourceGroup, internalSubnet)
if err != nil {
return nil, errors.Annotatef(err, "querying private IP addresses")
}
// Create a primary NIC for the machine. This needs to be static, so
// that we can create security rules that don't become invalid.
logger.Debugf("- creating primary NIC")
ipConfigurations := []network.InterfaceIPConfiguration{{
Name: to.StringPtr("primary"),
Properties: &network.InterfaceIPConfigurationPropertiesFormat{
PrivateIPAddress: to.StringPtr(privateIPAddress),
PrivateIPAllocationMethod: network.Static,
Subnet: &network.SubResource{internalSubnet.ID},
PublicIPAddress: &network.SubResource{publicIPAddress.ID},
},
}}
primaryNicName := vmName + "-primary"
primaryNicParams := network.Interface{
Location: to.StringPtr(location),
Tags: toTagsPtr(tags),
Properties: &network.InterfacePropertiesFormat{
IPConfigurations: &ipConfigurations,
},
}
primaryNic, err := nicClient.CreateOrUpdate(resourceGroup, primaryNicName, primaryNicParams)
if err != nil {
return nil, errors.Annotatef(err, "creating network interface for %q", vmName)
}
// Create a network security rule for the machine if we need to open
// the API server port.
if apiPort != nil {
logger.Debugf("- querying network security group")
securityGroupClient := network.SecurityGroupsClient{client}
securityGroupName := internalSecurityGroupName
securityGroup, err := securityGroupClient.Get(resourceGroup, securityGroupName)
if err != nil {
return nil, errors.Annotate(err, "querying network security group")
}
// NOTE(axw) this looks like TOCTTOU race territory, but it's
// safe because we only allocate/deallocate rules in this
// range during machine (de)provisioning, which is managed by
// a single goroutine. Non-internal ports are managed by the
// firewaller exclusively.
nextPriority, err := nextSecurityRulePriority(
securityGroup,
securityRuleInternalSSHInbound+1,
securityRuleInternalMax,
)
if err != nil {
return nil, errors.Trace(err)
}
apiSecurityRuleName := fmt.Sprintf("%s-api", vmName)
apiSecurityRule := network.SecurityRule{
Name: to.StringPtr(apiSecurityRuleName),
Properties: &network.SecurityRulePropertiesFormat{
Description: to.StringPtr("Allow API access to server machines"),
Protocol: network.SecurityRuleProtocolTCP,
SourceAddressPrefix: to.StringPtr("*"),
SourcePortRange: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr(privateIPAddress),
DestinationPortRange: to.StringPtr(fmt.Sprint(*apiPort)),
Access: network.Allow,
Priority: to.IntPtr(nextPriority),
Direction: network.Inbound,
},
}
logger.Debugf("- creating API network security rule")
securityRuleClient := network.SecurityRulesClient{client}
_, err = securityRuleClient.CreateOrUpdate(
resourceGroup, securityGroupName, apiSecurityRuleName, apiSecurityRule,
)
if err != nil {
return nil, errors.Annotate(err, "creating API network security rule")
}
}
// For now we only attach a single, flat network to each machine.
networkInterfaces := []compute.NetworkInterfaceReference{{
ID: primaryNic.ID,
Properties: &compute.NetworkInterfaceReferenceProperties{
Primary: to.BoolPtr(true),
},
}}
return &compute.NetworkProfile{&networkInterfaces}, nil
}
// createInternalSubnet creates an internal subnet for the specified resource group,
// within the specified virtual network.
//
// NOTE(axw) this method expects an up-to-date VirtualNetwork, and expects that are
// no concurrent subnet additions to the virtual network. At the moment we have only
// three places where we modify subnets: at bootstrap, when a new environment is
// created, and when an environment is destroyed.
func createInternalSubnet(
client network.ManagementClient,
resourceGroup string,
vnet *network.VirtualNetwork,
location string,
tags map[string]string,
) (*network.Subnet, error) {
nextAddressPrefix := (*vnet.Properties.AddressSpace.AddressPrefixes)[0]
if vnet.Properties.Subnets != nil {
if len(*vnet.Properties.Subnets) == len(*vnet.Properties.AddressSpace.AddressPrefixes) {
return nil, errors.Errorf(
"no available address prefixes in vnet %q",
to.String(vnet.Name),
)
}
addressPrefixesInUse := make(set.Strings)
for _, subnet := range *vnet.Properties.Subnets {
addressPrefixesInUse.Add(to.String(subnet.Properties.AddressPrefix))
}
for _, addressPrefix := range *vnet.Properties.AddressSpace.AddressPrefixes {
if !addressPrefixesInUse.Contains(addressPrefix) {
nextAddressPrefix = addressPrefix
break
}
}
}
// Create a network security group for the environment. There is only
// one NSG per environment (there's a limit of 100 per subscription),
// in which we manage rules for each exposed machine.
securityRules := []network.SecurityRule{sshSecurityRule}
securityGroupParams := network.SecurityGroup{
Location: to.StringPtr(location),
Tags: toTagsPtr(tags),
Properties: &network.SecurityGroupPropertiesFormat{
SecurityRules: &securityRules,
},
}
securityGroupClient := network.SecurityGroupsClient{client}
securityGroupName := internalSecurityGroupName
logger.Debugf("creating security group %q", securityGroupName)
nsg, err := securityGroupClient.CreateOrUpdate(
resourceGroup, securityGroupName, securityGroupParams,
)
if err != nil {
return nil, errors.Annotatef(err, "creating security group %q", securityGroupName)
}
// Now create a subnet with the next available address prefix, and
// associate the subnet with the NSG created above.
subnetName := internalSubnetName
subnetParams := network.Subnet{
Properties: &network.SubnetPropertiesFormat{
AddressPrefix: to.StringPtr(nextAddressPrefix),
NetworkSecurityGroup: &network.SubResource{nsg.ID},
},
}
logger.Debugf("creating subnet %q (%s)", subnetName, nextAddressPrefix)
subnetClient := network.SubnetsClient{client}
subnet, err := subnetClient.CreateOrUpdate(
resourceGroup, internalNetworkName, subnetName, subnetParams,
)
if err != nil {
return nil, errors.Annotatef(err, "creating subnet %q", subnetName)
}
return &subnet, nil
}
// Ports is specified in the Instance interface.
func (inst *azureInstance) Ports(machineId string) (ports []jujunetwork.PortRange, err error) {
inst.env.mu.Lock()
nsgClient := network.SecurityGroupsClient{inst.env.network}
inst.env.mu.Unlock()
securityGroupName := internalSecurityGroupName
nsg, err := nsgClient.Get(inst.env.resourceGroup, securityGroupName)
if err != nil {
return nil, errors.Annotate(err, "querying network security group")
}
if nsg.Properties.SecurityRules == nil {
return nil, nil
}
vmName := resourceName(names.NewMachineTag(machineId))
prefix := instanceNetworkSecurityRulePrefix(instance.Id(vmName))
for _, rule := range *nsg.Properties.SecurityRules {
if rule.Properties.Direction != network.Inbound {
continue
}
if rule.Properties.Access != network.Allow {
continue
}
if to.Int(rule.Properties.Priority) <= securityRuleInternalMax {
continue
}
if !strings.HasPrefix(to.String(rule.Name), prefix) {
continue
}
var portRange jujunetwork.PortRange
if *rule.Properties.DestinationPortRange == "*" {
portRange.FromPort = 0
portRange.ToPort = 65535
} else {
portRange, err = jujunetwork.ParsePortRange(
*rule.Properties.DestinationPortRange,
)
if err != nil {
return nil, errors.Annotatef(
err, "parsing port range for security rule %q",
to.String(rule.Name),
)
}
}
var protocols []string
switch rule.Properties.Protocol {
case network.SecurityRuleProtocolTCP:
protocols = []string{"tcp"}
case network.SecurityRuleProtocolUDP:
protocols = []string{"udp"}
default:
protocols = []string{"tcp", "udp"}
}
for _, protocol := range protocols {
portRange.Protocol = protocol
ports = append(ports, portRange)
}
}
return ports, nil
}
// OpenPorts is specified in the Instance interface.
func (inst *azureInstance) OpenPorts(machineId string, ports []jujunetwork.PortRange) error {
inst.env.mu.Lock()
nsgClient := network.SecurityGroupsClient{inst.env.network}
securityRuleClient := network.SecurityRulesClient{inst.env.network}
inst.env.mu.Unlock()
internalNetworkAddress, err := inst.internalNetworkAddress()
if err != nil {
return errors.Trace(err)
}
securityGroupName := internalSecurityGroupName
nsg, err := nsgClient.Get(inst.env.resourceGroup, securityGroupName)
if err != nil {
return errors.Annotate(err, "querying network security group")
}
var securityRules []network.SecurityRule
if nsg.Properties.SecurityRules != nil {
securityRules = *nsg.Properties.SecurityRules
} else {
nsg.Properties.SecurityRules = &securityRules
}
// Create rules one at a time; this is necessary to avoid trampling
// on changes made by the provisioner. We still record rules in the
// NSG in memory, so we can easily tell which priorities are available.
vmName := resourceName(names.NewMachineTag(machineId))
prefix := instanceNetworkSecurityRulePrefix(instance.Id(vmName))
for _, ports := range ports {
ruleName := securityRuleName(prefix, ports)
// Check if the rule already exists; OpenPorts must be idempotent.
var found bool
for _, rule := range securityRules {
if to.String(rule.Name) == ruleName {
found = true
break
}
}
if found {
logger.Debugf("security rule %q already exists", ruleName)
continue
}
logger.Debugf("creating security rule %q", ruleName)
priority, err := nextSecurityRulePriority(nsg, securityRuleInternalMax+1, securityRuleMax)
if err != nil {
return errors.Annotatef(err, "getting security rule priority for %s", ports)
}
var protocol network.SecurityRuleProtocol
switch ports.Protocol {
case "tcp":
protocol = network.SecurityRuleProtocolTCP
case "udp":
protocol = network.SecurityRuleProtocolUDP
default:
return errors.Errorf("invalid protocol %q", ports.Protocol)
}
var portRange string
if ports.FromPort != ports.ToPort {
portRange = fmt.Sprintf("%d-%d", ports.FromPort, ports.ToPort)
} else {
portRange = fmt.Sprint(ports.FromPort)
}
rule := network.SecurityRule{
Properties: &network.SecurityRulePropertiesFormat{
Description: to.StringPtr(ports.String()),
Protocol: protocol,
SourcePortRange: to.StringPtr("*"),
DestinationPortRange: to.StringPtr(portRange),
SourceAddressPrefix: to.StringPtr("*"),
DestinationAddressPrefix: to.StringPtr(internalNetworkAddress.Value),
Access: network.Allow,
Priority: to.IntPtr(priority),
Direction: network.Inbound,
},
}
if _, err := securityRuleClient.CreateOrUpdate(
inst.env.resourceGroup, securityGroupName, ruleName, rule,
); err != nil {
return errors.Annotatef(err, "creating security rule for %s", ports)
}
securityRules = append(securityRules, rule)
}
return nil
}
// createInternalSubnet creates an internal subnet for the specified resource group,
// within the specified virtual network.
//
// Subnets are tied to the resource group of the virtual network, so we must create
// them all in the controller resource group. We create the network security group
// for the subnet in the environment's resource group.
//
// NOTE(axw) this method expects an up-to-date VirtualNetwork, and expects that are
// no concurrent subnet additions to the virtual network. At the moment we have only
// three places where we modify subnets: at bootstrap, when a new environment is
// created, and when an environment is destroyed.
func createInternalSubnet(
client network.ManagementClient,
resourceGroup, controllerResourceGroup string,
vnet *network.VirtualNetwork,
location string,
tags map[string]string,
) (*network.Subnet, error) {
nextAddressPrefix := (*vnet.Properties.AddressSpace.AddressPrefixes)[0]
if vnet.Properties.Subnets != nil {
if len(*vnet.Properties.Subnets) == len(*vnet.Properties.AddressSpace.AddressPrefixes) {
return nil, errors.Errorf(
"no available address prefixes in vnet %q",
to.String(vnet.Name),
)
}
addressPrefixesInUse := make(set.Strings)
for _, subnet := range *vnet.Properties.Subnets {
addressPrefixesInUse.Add(to.String(subnet.Properties.AddressPrefix))
}
for _, addressPrefix := range *vnet.Properties.AddressSpace.AddressPrefixes {
if !addressPrefixesInUse.Contains(addressPrefix) {
nextAddressPrefix = addressPrefix
break
}
}
}
// Create a network security group for the environment. There is only
// one NSG per environment (there's a limit of 100 per subscription),
// in which we manage rules for each exposed machine.
securityRules := []network.SecurityRule{sshSecurityRule}
securityGroupParams := network.SecurityGroup{
Location: to.StringPtr(location),
Tags: toTagsPtr(tags),
Properties: &network.SecurityGroupPropertiesFormat{
SecurityRules: &securityRules,
},
}
securityGroupClient := network.SecurityGroupsClient{client}
securityGroupName := internalSecurityGroupName
logger.Debugf("creating security group %q", securityGroupName)
_, err := securityGroupClient.CreateOrUpdate(
resourceGroup, securityGroupName, securityGroupParams,
)
if err != nil {
return nil, errors.Annotatef(err, "creating security group %q", securityGroupName)
}
// Now create a subnet with the next available address prefix. The
// subnet must be created in the controller resource group, as it
// must be co-located with the vnet.
subnetName := resourceGroup
subnetParams := network.Subnet{
Properties: &network.SubnetPropertiesFormat{
AddressPrefix: to.StringPtr(nextAddressPrefix),
// NOTE(axw) we do NOT want to set the network security
// group as default for the subnet, because that will
// create a dependency from the controller resource
// group to environment resource groups. Instead, we
// set the NSG on NICs.
},
}
logger.Debugf("creating subnet %q (%s)", subnetName, nextAddressPrefix)
subnetClient := network.SubnetsClient{client}
subnet, err := subnetClient.CreateOrUpdate(
controllerResourceGroup, internalNetworkName, subnetName, subnetParams,
)
if err != nil {
return nil, errors.Annotatef(err, "creating subnet %q", subnetName)
}
return &subnet, nil
}