// deleteInstanceNetworkSecurityRules deletes network security rules in the // internal network security group that correspond to the specified machine. // // This is expected to delete *all* security rules related to the instance, // i.e. both the ones opened by OpenPorts above, and the ones opened for API // access. func deleteInstanceNetworkSecurityRules( resourceGroup string, id instance.Id, nsgClient network.SecurityGroupsClient, securityRuleClient network.SecurityRulesClient, ) error { nsg, err := nsgClient.Get(resourceGroup, internalSecurityGroupName) if err != nil { return errors.Annotate(err, "querying network security group") } if nsg.Properties.SecurityRules == nil { return nil } prefix := instanceNetworkSecurityRulePrefix(id) for _, rule := range *nsg.Properties.SecurityRules { ruleName := to.String(rule.Name) if !strings.HasPrefix(ruleName, prefix) { continue } result, err := securityRuleClient.Delete( resourceGroup, internalSecurityGroupName, ruleName, ) if err != nil { if result.Response == nil || result.StatusCode != http.StatusNotFound { return errors.Annotatef(err, "deleting security rule %q", ruleName) } } } return nil }
// deleteInstanceNetworkSecurityRules deletes network security rules in the // internal network security group that correspond to the specified machine. // // This is expected to delete *all* security rules related to the instance, // i.e. both the ones opened by OpenPorts above, and the ones opened for API // access. func deleteInstanceNetworkSecurityRules( resourceGroup string, id instance.Id, nsgClient network.SecurityGroupsClient, securityRuleClient network.SecurityRulesClient, callAPI callAPIFunc, ) error { var nsg network.SecurityGroup if err := callAPI(func() (autorest.Response, error) { var err error nsg, err = nsgClient.Get(resourceGroup, internalSecurityGroupName, "") return nsg.Response, err }); err != nil { return errors.Annotate(err, "querying network security group") } if nsg.Properties.SecurityRules == nil { return nil } prefix := instanceNetworkSecurityRulePrefix(id) for _, rule := range *nsg.Properties.SecurityRules { ruleName := to.String(rule.Name) if !strings.HasPrefix(ruleName, prefix) { continue } var result autorest.Response err := callAPI(func() (autorest.Response, error) { var err error result, err = securityRuleClient.Delete( resourceGroup, internalSecurityGroupName, ruleName, nil, // abort channel ) return result, err }) if err != nil { if result.Response == nil || result.StatusCode != http.StatusNotFound { return errors.Annotatef(err, "deleting security rule %q", ruleName) } } } return nil }
func newNetworkProfile( client network.ManagementClient, vmName string, apiPort *int, internalSubnet *network.Subnet, resourceGroup string, location string, tags map[string]string, ) (*compute.NetworkProfile, error) { logger.Debugf("creating network profile for %q", vmName) // Create a public IP for the NIC. Public IP addresses are dynamic. logger.Debugf("- allocating public IP address") pipClient := network.PublicIPAddressesClient{client} publicIPAddressParams := network.PublicIPAddress{ Location: to.StringPtr(location), Tags: toTagsPtr(tags), Properties: &network.PublicIPAddressPropertiesFormat{ PublicIPAllocationMethod: network.Dynamic, }, } publicIPAddressName := vmName + "-public-ip" publicIPAddress, err := pipClient.CreateOrUpdate(resourceGroup, publicIPAddressName, publicIPAddressParams) if err != nil { return nil, errors.Annotatef(err, "creating public IP address for %q", vmName) } // Determine the next available private IP address. nicClient := network.InterfacesClient{client} privateIPAddress, err := nextSubnetIPAddress(nicClient, resourceGroup, internalSubnet) if err != nil { return nil, errors.Annotatef(err, "querying private IP addresses") } // Create a primary NIC for the machine. This needs to be static, so // that we can create security rules that don't become invalid. logger.Debugf("- creating primary NIC") ipConfigurations := []network.InterfaceIPConfiguration{{ Name: to.StringPtr("primary"), Properties: &network.InterfaceIPConfigurationPropertiesFormat{ PrivateIPAddress: to.StringPtr(privateIPAddress), PrivateIPAllocationMethod: network.Static, Subnet: &network.SubResource{internalSubnet.ID}, PublicIPAddress: &network.SubResource{publicIPAddress.ID}, }, }} primaryNicName := vmName + "-primary" primaryNicParams := network.Interface{ Location: to.StringPtr(location), Tags: toTagsPtr(tags), Properties: &network.InterfacePropertiesFormat{ IPConfigurations: &ipConfigurations, }, } primaryNic, err := nicClient.CreateOrUpdate(resourceGroup, primaryNicName, primaryNicParams) if err != nil { return nil, errors.Annotatef(err, "creating network interface for %q", vmName) } // Create a network security rule for the machine if we need to open // the API server port. if apiPort != nil { logger.Debugf("- querying network security group") securityGroupClient := network.SecurityGroupsClient{client} securityGroupName := internalSecurityGroupName securityGroup, err := securityGroupClient.Get(resourceGroup, securityGroupName) if err != nil { return nil, errors.Annotate(err, "querying network security group") } // NOTE(axw) this looks like TOCTTOU race territory, but it's // safe because we only allocate/deallocate rules in this // range during machine (de)provisioning, which is managed by // a single goroutine. Non-internal ports are managed by the // firewaller exclusively. nextPriority, err := nextSecurityRulePriority( securityGroup, securityRuleInternalSSHInbound+1, securityRuleInternalMax, ) if err != nil { return nil, errors.Trace(err) } apiSecurityRuleName := fmt.Sprintf("%s-api", vmName) apiSecurityRule := network.SecurityRule{ Name: to.StringPtr(apiSecurityRuleName), Properties: &network.SecurityRulePropertiesFormat{ Description: to.StringPtr("Allow API access to server machines"), Protocol: network.SecurityRuleProtocolTCP, SourceAddressPrefix: to.StringPtr("*"), SourcePortRange: to.StringPtr("*"), DestinationAddressPrefix: to.StringPtr(privateIPAddress), DestinationPortRange: to.StringPtr(fmt.Sprint(*apiPort)), Access: network.Allow, Priority: to.IntPtr(nextPriority), Direction: network.Inbound, }, } logger.Debugf("- creating API network security rule") securityRuleClient := network.SecurityRulesClient{client} _, err = securityRuleClient.CreateOrUpdate( resourceGroup, securityGroupName, apiSecurityRuleName, apiSecurityRule, ) if err != nil { return nil, errors.Annotate(err, "creating API network security rule") } } // For now we only attach a single, flat network to each machine. networkInterfaces := []compute.NetworkInterfaceReference{{ ID: primaryNic.ID, Properties: &compute.NetworkInterfaceReferenceProperties{ Primary: to.BoolPtr(true), }, }} return &compute.NetworkProfile{&networkInterfaces}, nil }
// createInternalSubnet creates an internal subnet for the specified resource group, // within the specified virtual network. // // NOTE(axw) this method expects an up-to-date VirtualNetwork, and expects that are // no concurrent subnet additions to the virtual network. At the moment we have only // three places where we modify subnets: at bootstrap, when a new environment is // created, and when an environment is destroyed. func createInternalSubnet( client network.ManagementClient, resourceGroup string, vnet *network.VirtualNetwork, location string, tags map[string]string, ) (*network.Subnet, error) { nextAddressPrefix := (*vnet.Properties.AddressSpace.AddressPrefixes)[0] if vnet.Properties.Subnets != nil { if len(*vnet.Properties.Subnets) == len(*vnet.Properties.AddressSpace.AddressPrefixes) { return nil, errors.Errorf( "no available address prefixes in vnet %q", to.String(vnet.Name), ) } addressPrefixesInUse := make(set.Strings) for _, subnet := range *vnet.Properties.Subnets { addressPrefixesInUse.Add(to.String(subnet.Properties.AddressPrefix)) } for _, addressPrefix := range *vnet.Properties.AddressSpace.AddressPrefixes { if !addressPrefixesInUse.Contains(addressPrefix) { nextAddressPrefix = addressPrefix break } } } // Create a network security group for the environment. There is only // one NSG per environment (there's a limit of 100 per subscription), // in which we manage rules for each exposed machine. securityRules := []network.SecurityRule{sshSecurityRule} securityGroupParams := network.SecurityGroup{ Location: to.StringPtr(location), Tags: toTagsPtr(tags), Properties: &network.SecurityGroupPropertiesFormat{ SecurityRules: &securityRules, }, } securityGroupClient := network.SecurityGroupsClient{client} securityGroupName := internalSecurityGroupName logger.Debugf("creating security group %q", securityGroupName) nsg, err := securityGroupClient.CreateOrUpdate( resourceGroup, securityGroupName, securityGroupParams, ) if err != nil { return nil, errors.Annotatef(err, "creating security group %q", securityGroupName) } // Now create a subnet with the next available address prefix, and // associate the subnet with the NSG created above. subnetName := internalSubnetName subnetParams := network.Subnet{ Properties: &network.SubnetPropertiesFormat{ AddressPrefix: to.StringPtr(nextAddressPrefix), NetworkSecurityGroup: &network.SubResource{nsg.ID}, }, } logger.Debugf("creating subnet %q (%s)", subnetName, nextAddressPrefix) subnetClient := network.SubnetsClient{client} subnet, err := subnetClient.CreateOrUpdate( resourceGroup, internalNetworkName, subnetName, subnetParams, ) if err != nil { return nil, errors.Annotatef(err, "creating subnet %q", subnetName) } return &subnet, nil }
// Ports is specified in the Instance interface. func (inst *azureInstance) Ports(machineId string) (ports []jujunetwork.PortRange, err error) { inst.env.mu.Lock() nsgClient := network.SecurityGroupsClient{inst.env.network} inst.env.mu.Unlock() securityGroupName := internalSecurityGroupName nsg, err := nsgClient.Get(inst.env.resourceGroup, securityGroupName) if err != nil { return nil, errors.Annotate(err, "querying network security group") } if nsg.Properties.SecurityRules == nil { return nil, nil } vmName := resourceName(names.NewMachineTag(machineId)) prefix := instanceNetworkSecurityRulePrefix(instance.Id(vmName)) for _, rule := range *nsg.Properties.SecurityRules { if rule.Properties.Direction != network.Inbound { continue } if rule.Properties.Access != network.Allow { continue } if to.Int(rule.Properties.Priority) <= securityRuleInternalMax { continue } if !strings.HasPrefix(to.String(rule.Name), prefix) { continue } var portRange jujunetwork.PortRange if *rule.Properties.DestinationPortRange == "*" { portRange.FromPort = 0 portRange.ToPort = 65535 } else { portRange, err = jujunetwork.ParsePortRange( *rule.Properties.DestinationPortRange, ) if err != nil { return nil, errors.Annotatef( err, "parsing port range for security rule %q", to.String(rule.Name), ) } } var protocols []string switch rule.Properties.Protocol { case network.SecurityRuleProtocolTCP: protocols = []string{"tcp"} case network.SecurityRuleProtocolUDP: protocols = []string{"udp"} default: protocols = []string{"tcp", "udp"} } for _, protocol := range protocols { portRange.Protocol = protocol ports = append(ports, portRange) } } return ports, nil }
// OpenPorts is specified in the Instance interface. func (inst *azureInstance) OpenPorts(machineId string, ports []jujunetwork.PortRange) error { inst.env.mu.Lock() nsgClient := network.SecurityGroupsClient{inst.env.network} securityRuleClient := network.SecurityRulesClient{inst.env.network} inst.env.mu.Unlock() internalNetworkAddress, err := inst.internalNetworkAddress() if err != nil { return errors.Trace(err) } securityGroupName := internalSecurityGroupName nsg, err := nsgClient.Get(inst.env.resourceGroup, securityGroupName) if err != nil { return errors.Annotate(err, "querying network security group") } var securityRules []network.SecurityRule if nsg.Properties.SecurityRules != nil { securityRules = *nsg.Properties.SecurityRules } else { nsg.Properties.SecurityRules = &securityRules } // Create rules one at a time; this is necessary to avoid trampling // on changes made by the provisioner. We still record rules in the // NSG in memory, so we can easily tell which priorities are available. vmName := resourceName(names.NewMachineTag(machineId)) prefix := instanceNetworkSecurityRulePrefix(instance.Id(vmName)) for _, ports := range ports { ruleName := securityRuleName(prefix, ports) // Check if the rule already exists; OpenPorts must be idempotent. var found bool for _, rule := range securityRules { if to.String(rule.Name) == ruleName { found = true break } } if found { logger.Debugf("security rule %q already exists", ruleName) continue } logger.Debugf("creating security rule %q", ruleName) priority, err := nextSecurityRulePriority(nsg, securityRuleInternalMax+1, securityRuleMax) if err != nil { return errors.Annotatef(err, "getting security rule priority for %s", ports) } var protocol network.SecurityRuleProtocol switch ports.Protocol { case "tcp": protocol = network.SecurityRuleProtocolTCP case "udp": protocol = network.SecurityRuleProtocolUDP default: return errors.Errorf("invalid protocol %q", ports.Protocol) } var portRange string if ports.FromPort != ports.ToPort { portRange = fmt.Sprintf("%d-%d", ports.FromPort, ports.ToPort) } else { portRange = fmt.Sprint(ports.FromPort) } rule := network.SecurityRule{ Properties: &network.SecurityRulePropertiesFormat{ Description: to.StringPtr(ports.String()), Protocol: protocol, SourcePortRange: to.StringPtr("*"), DestinationPortRange: to.StringPtr(portRange), SourceAddressPrefix: to.StringPtr("*"), DestinationAddressPrefix: to.StringPtr(internalNetworkAddress.Value), Access: network.Allow, Priority: to.IntPtr(priority), Direction: network.Inbound, }, } if _, err := securityRuleClient.CreateOrUpdate( inst.env.resourceGroup, securityGroupName, ruleName, rule, ); err != nil { return errors.Annotatef(err, "creating security rule for %s", ports) } securityRules = append(securityRules, rule) } return nil }
// createInternalSubnet creates an internal subnet for the specified resource group, // within the specified virtual network. // // Subnets are tied to the resource group of the virtual network, so we must create // them all in the controller resource group. We create the network security group // for the subnet in the environment's resource group. // // NOTE(axw) this method expects an up-to-date VirtualNetwork, and expects that are // no concurrent subnet additions to the virtual network. At the moment we have only // three places where we modify subnets: at bootstrap, when a new environment is // created, and when an environment is destroyed. func createInternalSubnet( client network.ManagementClient, resourceGroup, controllerResourceGroup string, vnet *network.VirtualNetwork, location string, tags map[string]string, ) (*network.Subnet, error) { nextAddressPrefix := (*vnet.Properties.AddressSpace.AddressPrefixes)[0] if vnet.Properties.Subnets != nil { if len(*vnet.Properties.Subnets) == len(*vnet.Properties.AddressSpace.AddressPrefixes) { return nil, errors.Errorf( "no available address prefixes in vnet %q", to.String(vnet.Name), ) } addressPrefixesInUse := make(set.Strings) for _, subnet := range *vnet.Properties.Subnets { addressPrefixesInUse.Add(to.String(subnet.Properties.AddressPrefix)) } for _, addressPrefix := range *vnet.Properties.AddressSpace.AddressPrefixes { if !addressPrefixesInUse.Contains(addressPrefix) { nextAddressPrefix = addressPrefix break } } } // Create a network security group for the environment. There is only // one NSG per environment (there's a limit of 100 per subscription), // in which we manage rules for each exposed machine. securityRules := []network.SecurityRule{sshSecurityRule} securityGroupParams := network.SecurityGroup{ Location: to.StringPtr(location), Tags: toTagsPtr(tags), Properties: &network.SecurityGroupPropertiesFormat{ SecurityRules: &securityRules, }, } securityGroupClient := network.SecurityGroupsClient{client} securityGroupName := internalSecurityGroupName logger.Debugf("creating security group %q", securityGroupName) _, err := securityGroupClient.CreateOrUpdate( resourceGroup, securityGroupName, securityGroupParams, ) if err != nil { return nil, errors.Annotatef(err, "creating security group %q", securityGroupName) } // Now create a subnet with the next available address prefix. The // subnet must be created in the controller resource group, as it // must be co-located with the vnet. subnetName := resourceGroup subnetParams := network.Subnet{ Properties: &network.SubnetPropertiesFormat{ AddressPrefix: to.StringPtr(nextAddressPrefix), // NOTE(axw) we do NOT want to set the network security // group as default for the subnet, because that will // create a dependency from the controller resource // group to environment resource groups. Instead, we // set the NSG on NICs. }, } logger.Debugf("creating subnet %q (%s)", subnetName, nextAddressPrefix) subnetClient := network.SubnetsClient{client} subnet, err := subnetClient.CreateOrUpdate( controllerResourceGroup, internalNetworkName, subnetName, subnetParams, ) if err != nil { return nil, errors.Annotatef(err, "creating subnet %q", subnetName) } return &subnet, nil }