// Validate allocation of a nodePort when the externalTraffic=OnlyLocal annotation is set
// and type is LoadBalancer
func TestServiceRegistryExternalTrafficAnnotationHealthCheckNodePortAllocation(t *testing.T) {
ctx := genericapirequest.NewDefaultContext()
storage, _ := NewTestREST(t, nil)
svc := &api.Service{
ObjectMeta: metav1.ObjectMeta{Name: "external-lb-esipp",
Annotations: map[string]string{
service.BetaAnnotationExternalTraffic: service.AnnotationValueExternalTrafficLocal,
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{"bar": "baz"},
SessionAffinity: api.ServiceAffinityNone,
Type: api.ServiceTypeLoadBalancer,
Ports: []api.ServicePort{{
Port: 6502,
Protocol: api.ProtocolTCP,
TargetPort: intstr.FromInt(6502),
}},
},
}
created_svc, err := storage.Create(ctx, svc)
if created_svc == nil || err != nil {
t.Errorf("Unexpected failure creating service %v", err)
}
created_service := created_svc.(*api.Service)
if !service.NeedsHealthCheck(created_service) {
t.Errorf("Unexpected missing annotation %s", service.BetaAnnotationExternalTraffic)
}
port := service.GetServiceHealthCheckNodePort(created_service)
if port == 0 {
t.Errorf("Failed to allocate and create the health check node port annotation %s", service.BetaAnnotationHealthCheckNodePort)
}
}
// Validate that the health check nodePort is not allocated when the externalTraffic annotation is !"OnlyLocal"
func TestServiceRegistryExternalTrafficAnnotationGlobal(t *testing.T) {
ctx := api.NewDefaultContext()
storage, _ := NewTestREST(t, nil)
svc := &api.Service{
ObjectMeta: api.ObjectMeta{Name: "external-lb-esipp",
Annotations: map[string]string{
service.AnnotationExternalTraffic: service.AnnotationValueExternalTrafficGlobal,
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{"bar": "baz"},
SessionAffinity: api.ServiceAffinityNone,
Type: api.ServiceTypeLoadBalancer,
Ports: []api.ServicePort{{
Port: 6502,
Protocol: api.ProtocolTCP,
TargetPort: intstr.FromInt(6502),
}},
},
}
created_svc, err := storage.Create(ctx, svc)
if created_svc == nil || err != nil {
t.Errorf("Unexpected failure creating service %v", err)
}
created_service := created_svc.(*api.Service)
// Make sure the service does not have the annotation
if service.NeedsHealthCheck(created_service) {
t.Errorf("Unexpected value for annotation %s", service.AnnotationExternalTraffic)
}
// Make sure the service does not have the health check node port allocated
port := service.GetServiceHealthCheckNodePort(created_service)
if port != 0 {
t.Errorf("Unexpected allocation of health check node port annotation %s", service.AnnotationHealthCheckNodePort)
}
}
func shouldCheckOrAssignHealthCheckNodePort(service *api.Service) bool {
if service.Spec.Type == api.ServiceTypeLoadBalancer {
// True if Service-type == LoadBalancer AND annotation AnnotationExternalTraffic present
return (featuregate.DefaultFeatureGate.ExternalTrafficLocalOnly() && apiservice.NeedsHealthCheck(service))
}
glog.V(4).Infof("Service type: %v does not need health check node port", service.Spec.Type)
return false
}
func validateLoadBalancer(t *testing.T, loadBalancer network.LoadBalancer, services ...api.Service) {
expectedRuleCount := 0
for _, svc := range services {
for _, wantedRule := range svc.Spec.Ports {
expectedRuleCount++
wantedRuleName := getRuleName(&svc, wantedRule)
foundRule := false
for _, actualRule := range *loadBalancer.Properties.LoadBalancingRules {
if strings.EqualFold(*actualRule.Name, wantedRuleName) &&
*actualRule.Properties.FrontendPort == wantedRule.Port &&
*actualRule.Properties.BackendPort == wantedRule.Port {
foundRule = true
break
}
}
if !foundRule {
t.Errorf("Expected load balancer rule but didn't find it: %q", wantedRuleName)
}
foundProbe := false
if serviceapi.NeedsHealthCheck(&svc) {
path, port := serviceapi.GetServiceHealthCheckPathPort(&svc)
for _, actualProbe := range *loadBalancer.Properties.Probes {
if strings.EqualFold(*actualProbe.Name, wantedRuleName) &&
*actualProbe.Properties.Port == port &&
*actualProbe.Properties.RequestPath == path &&
actualProbe.Properties.Protocol == network.ProbeProtocolHTTP {
foundProbe = true
break
}
}
} else {
for _, actualProbe := range *loadBalancer.Properties.Probes {
if strings.EqualFold(*actualProbe.Name, wantedRuleName) &&
*actualProbe.Properties.Port == wantedRule.NodePort {
foundProbe = true
break
}
}
}
if !foundProbe {
for _, actualProbe := range *loadBalancer.Properties.Probes {
t.Logf("Probe: %s %d", *actualProbe.Name, *actualProbe.Properties.Port)
}
t.Errorf("Expected loadbalancer probe but didn't find it: %q", wantedRuleName)
}
}
}
lenRules := len(*loadBalancer.Properties.LoadBalancingRules)
if lenRules != expectedRuleCount {
t.Errorf("Expected the loadbalancer to have %d rules. Found %d.", expectedRuleCount, lenRules)
}
lenProbes := len(*loadBalancer.Properties.Probes)
if lenProbes != expectedRuleCount {
t.Errorf("Expected the loadbalancer to have %d probes. Found %d.", expectedRuleCount, lenProbes)
}
}
// OnServiceUpdate tracks the active set of service proxies.
// They will be synchronized using syncProxyRules()
func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) {
start := time.Now()
defer func() {
glog.V(4).Infof("OnServiceUpdate took %v for %d services", time.Since(start), len(allServices))
}()
proxier.mu.Lock()
defer proxier.mu.Unlock()
proxier.haveReceivedServiceUpdate = true
activeServices := make(map[proxy.ServicePortName]bool) // use a map as a set
for i := range allServices {
service := &allServices[i]
svcName := types.NamespacedName{
Namespace: service.Namespace,
Name: service.Name,
}
// if ClusterIP is "None" or empty, skip proxying
if !api.IsServiceIPSet(service) {
glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP)
continue
}
for i := range service.Spec.Ports {
servicePort := &service.Spec.Ports[i]
serviceName := proxy.ServicePortName{
NamespacedName: svcName,
Port: servicePort.Name,
}
activeServices[serviceName] = true
info, exists := proxier.serviceMap[serviceName]
if exists && proxier.sameConfig(info, service, servicePort) {
// Nothing changed.
continue
}
if exists {
// Something changed.
glog.V(3).Infof("Something changed for service %q: removing it", serviceName)
delete(proxier.serviceMap, serviceName)
}
serviceIP := net.ParseIP(service.Spec.ClusterIP)
glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, serviceIP, servicePort.Port, servicePort.Protocol)
info = newServiceInfo(serviceName)
info.clusterIP = serviceIP
info.port = int(servicePort.Port)
info.protocol = servicePort.Protocol
info.nodePort = int(servicePort.NodePort)
info.externalIPs = service.Spec.ExternalIPs
// Deep-copy in case the service instance changes
info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer)
info.sessionAffinityType = service.Spec.SessionAffinity
info.loadBalancerSourceRanges = service.Spec.LoadBalancerSourceRanges
info.onlyNodeLocalEndpoints = apiservice.NeedsHealthCheck(service) && featuregate.DefaultFeatureGate.ExternalTrafficLocalOnly()
if info.onlyNodeLocalEndpoints {
p := apiservice.GetServiceHealthCheckNodePort(service)
if p == 0 {
glog.Errorf("Service does not contain necessary annotation %v",
apiservice.AnnotationHealthCheckNodePort)
} else {
info.healthCheckNodePort = int(p)
// Turn on healthcheck responder to listen on the health check nodePort
healthcheck.AddServiceListener(serviceName.NamespacedName, info.healthCheckNodePort)
}
}
proxier.serviceMap[serviceName] = info
glog.V(4).Infof("added serviceInfo(%s): %s", serviceName, spew.Sdump(info))
}
}
staleUDPServices := sets.NewString()
// Remove serviceports missing from the update.
for name, info := range proxier.serviceMap {
if !activeServices[name] {
glog.V(1).Infof("Removing service %q", name)
if info.protocol == api.ProtocolUDP {
staleUDPServices.Insert(info.clusterIP.String())
}
delete(proxier.serviceMap, name)
if info.onlyNodeLocalEndpoints && info.healthCheckNodePort > 0 {
// Remove ServiceListener health check nodePorts from the health checker
// TODO - Stats
healthcheck.DeleteServiceListener(name.NamespacedName, info.healthCheckNodePort)
}
}
}
proxier.syncProxyRules()
proxier.deleteServiceConnections(staleUDPServices.List())
}
// This ensures load balancer exists and the frontend ip config is setup.
// This also reconciles the Service's Ports with the LoadBalancer config.
// This entails adding rules/probes for expected Ports and removing stale rules/ports.
func (az *Cloud) reconcileLoadBalancer(lb network.LoadBalancer, pip *network.PublicIPAddress, clusterName string, service *api.Service, nodeNames []string) (network.LoadBalancer, bool, error) {
lbName := getLoadBalancerName(clusterName)
serviceName := getServiceName(service)
lbFrontendIPConfigName := getFrontendIPConfigName(service)
lbFrontendIPConfigID := az.getFrontendIPConfigID(lbName, lbFrontendIPConfigName)
lbBackendPoolName := getBackendPoolName(clusterName)
lbBackendPoolID := az.getBackendPoolID(lbName, lbBackendPoolName)
wantLb := len(service.Spec.Ports) > 0
dirtyLb := false
// Ensure LoadBalancer's Backend Pool Configuration
if wantLb {
if lb.Properties.BackendAddressPools == nil ||
len(*lb.Properties.BackendAddressPools) == 0 {
lb.Properties.BackendAddressPools = &[]network.BackendAddressPool{
{
Name: to.StringPtr(lbBackendPoolName),
},
}
glog.V(10).Infof("reconcile(%s)(%t): lb backendpool - adding", serviceName, wantLb)
dirtyLb = true
} else if len(*lb.Properties.BackendAddressPools) != 1 ||
!strings.EqualFold(*(*lb.Properties.BackendAddressPools)[0].Name, lbBackendPoolName) {
return lb, false, fmt.Errorf("loadbalancer is misconfigured with a different backend pool")
}
}
// Ensure LoadBalancer's Frontend IP Configurations
dirtyConfigs := false
newConfigs := []network.FrontendIPConfiguration{}
if lb.Properties.FrontendIPConfigurations != nil {
newConfigs = *lb.Properties.FrontendIPConfigurations
}
if !wantLb {
for i := len(newConfigs) - 1; i >= 0; i-- {
config := newConfigs[i]
if strings.EqualFold(*config.Name, lbFrontendIPConfigName) {
glog.V(3).Infof("reconcile(%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, lbFrontendIPConfigName)
newConfigs = append(newConfigs[:i], newConfigs[i+1:]...)
dirtyConfigs = true
}
}
} else {
foundConfig := false
for _, config := range newConfigs {
if strings.EqualFold(*config.Name, lbFrontendIPConfigName) {
foundConfig = true
break
}
}
if !foundConfig {
newConfigs = append(newConfigs,
network.FrontendIPConfiguration{
Name: to.StringPtr(lbFrontendIPConfigName),
Properties: &network.FrontendIPConfigurationPropertiesFormat{
PublicIPAddress: &network.PublicIPAddress{
ID: pip.ID,
},
},
})
glog.V(10).Infof("reconcile(%s)(%t): lb frontendconfig(%s) - adding", serviceName, wantLb, lbFrontendIPConfigName)
dirtyConfigs = true
}
}
if dirtyConfigs {
dirtyLb = true
lb.Properties.FrontendIPConfigurations = &newConfigs
}
// update probes/rules
expectedProbes := make([]network.Probe, len(service.Spec.Ports))
expectedRules := make([]network.LoadBalancingRule, len(service.Spec.Ports))
for i, port := range service.Spec.Ports {
lbRuleName := getRuleName(service, port)
transportProto, _, probeProto, err := getProtocolsFromKubernetesProtocol(port.Protocol)
if err != nil {
return lb, false, err
}
if serviceapi.NeedsHealthCheck(service) {
podPresencePath, podPresencePort := serviceapi.GetServiceHealthCheckPathPort(service)
expectedProbes[i] = network.Probe{
Name: &lbRuleName,
Properties: &network.ProbePropertiesFormat{
RequestPath: to.StringPtr(podPresencePath),
Protocol: network.ProbeProtocolHTTP,
Port: to.Int32Ptr(podPresencePort),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
}
} else {
expectedProbes[i] = network.Probe{
Name: &lbRuleName,
Properties: &network.ProbePropertiesFormat{
Protocol: probeProto,
Port: to.Int32Ptr(port.NodePort),
IntervalInSeconds: to.Int32Ptr(5),
NumberOfProbes: to.Int32Ptr(2),
},
}
}
expectedRules[i] = network.LoadBalancingRule{
Name: &lbRuleName,
Properties: &network.LoadBalancingRulePropertiesFormat{
Protocol: transportProto,
FrontendIPConfiguration: &network.SubResource{
ID: to.StringPtr(lbFrontendIPConfigID),
},
BackendAddressPool: &network.SubResource{
ID: to.StringPtr(lbBackendPoolID),
},
Probe: &network.SubResource{
ID: to.StringPtr(az.getLoadBalancerProbeID(lbName, lbRuleName)),
},
FrontendPort: to.Int32Ptr(port.Port),
BackendPort: to.Int32Ptr(port.Port),
EnableFloatingIP: to.BoolPtr(true),
},
}
}
// remove unwanted probes
dirtyProbes := false
var updatedProbes []network.Probe
if lb.Properties.Probes != nil {
updatedProbes = *lb.Properties.Probes
}
for i := len(updatedProbes) - 1; i >= 0; i-- {
existingProbe := updatedProbes[i]
if serviceOwnsRule(service, *existingProbe.Name) {
glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - considering evicting", serviceName, wantLb, *existingProbe.Name)
keepProbe := false
if findProbe(expectedProbes, existingProbe) {
glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - keeping", serviceName, wantLb, *existingProbe.Name)
keepProbe = true
}
if !keepProbe {
updatedProbes = append(updatedProbes[:i], updatedProbes[i+1:]...)
glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - dropping", serviceName, wantLb, *existingProbe.Name)
dirtyProbes = true
}
}
}
// add missing, wanted probes
for _, expectedProbe := range expectedProbes {
foundProbe := false
if findProbe(updatedProbes, expectedProbe) {
glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - already exists", serviceName, wantLb, *expectedProbe.Name)
foundProbe = true
}
if !foundProbe {
glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - adding", serviceName, wantLb, *expectedProbe.Name)
updatedProbes = append(updatedProbes, expectedProbe)
dirtyProbes = true
}
}
if dirtyProbes {
dirtyLb = true
lb.Properties.Probes = &updatedProbes
}
// update rules
dirtyRules := false
var updatedRules []network.LoadBalancingRule
if lb.Properties.LoadBalancingRules != nil {
updatedRules = *lb.Properties.LoadBalancingRules
}
// update rules: remove unwanted
for i := len(updatedRules) - 1; i >= 0; i-- {
existingRule := updatedRules[i]
if serviceOwnsRule(service, *existingRule.Name) {
keepRule := false
glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name)
if findRule(expectedRules, existingRule) {
glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - keeping", serviceName, wantLb, *existingRule.Name)
keepRule = true
}
if !keepRule {
glog.V(3).Infof("reconcile(%s)(%t): lb rule(%s) - dropping", serviceName, wantLb, *existingRule.Name)
updatedRules = append(updatedRules[:i], updatedRules[i+1:]...)
dirtyRules = true
}
}
}
// update rules: add needed
for _, expectedRule := range expectedRules {
foundRule := false
if findRule(updatedRules, expectedRule) {
glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name)
foundRule = true
}
if !foundRule {
glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) adding", serviceName, wantLb, *expectedRule.Name)
updatedRules = append(updatedRules, expectedRule)
dirtyRules = true
}
}
if dirtyRules {
dirtyLb = true
lb.Properties.LoadBalancingRules = &updatedRules
}
return lb, dirtyLb, nil
}
