// Validate allocation of a nodePort when the externalTraffic=OnlyLocal annotation is set // and type is LoadBalancer func TestServiceRegistryExternalTrafficAnnotationHealthCheckNodePortAllocation(t *testing.T) { ctx := genericapirequest.NewDefaultContext() storage, _ := NewTestREST(t, nil) svc := &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "external-lb-esipp", Annotations: map[string]string{ service.BetaAnnotationExternalTraffic: service.AnnotationValueExternalTrafficLocal, }, }, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeLoadBalancer, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, TargetPort: intstr.FromInt(6502), }}, }, } created_svc, err := storage.Create(ctx, svc) if created_svc == nil || err != nil { t.Errorf("Unexpected failure creating service %v", err) } created_service := created_svc.(*api.Service) if !service.NeedsHealthCheck(created_service) { t.Errorf("Unexpected missing annotation %s", service.BetaAnnotationExternalTraffic) } port := service.GetServiceHealthCheckNodePort(created_service) if port == 0 { t.Errorf("Failed to allocate and create the health check node port annotation %s", service.BetaAnnotationHealthCheckNodePort) } }
// Validate that the health check nodePort is not allocated when the externalTraffic annotation is !"OnlyLocal" func TestServiceRegistryExternalTrafficAnnotationGlobal(t *testing.T) { ctx := api.NewDefaultContext() storage, _ := NewTestREST(t, nil) svc := &api.Service{ ObjectMeta: api.ObjectMeta{Name: "external-lb-esipp", Annotations: map[string]string{ service.AnnotationExternalTraffic: service.AnnotationValueExternalTrafficGlobal, }, }, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, SessionAffinity: api.ServiceAffinityNone, Type: api.ServiceTypeLoadBalancer, Ports: []api.ServicePort{{ Port: 6502, Protocol: api.ProtocolTCP, TargetPort: intstr.FromInt(6502), }}, }, } created_svc, err := storage.Create(ctx, svc) if created_svc == nil || err != nil { t.Errorf("Unexpected failure creating service %v", err) } created_service := created_svc.(*api.Service) // Make sure the service does not have the annotation if service.NeedsHealthCheck(created_service) { t.Errorf("Unexpected value for annotation %s", service.AnnotationExternalTraffic) } // Make sure the service does not have the health check node port allocated port := service.GetServiceHealthCheckNodePort(created_service) if port != 0 { t.Errorf("Unexpected allocation of health check node port annotation %s", service.AnnotationHealthCheckNodePort) } }
func shouldCheckOrAssignHealthCheckNodePort(service *api.Service) bool { if service.Spec.Type == api.ServiceTypeLoadBalancer { // True if Service-type == LoadBalancer AND annotation AnnotationExternalTraffic present return (featuregate.DefaultFeatureGate.ExternalTrafficLocalOnly() && apiservice.NeedsHealthCheck(service)) } glog.V(4).Infof("Service type: %v does not need health check node port", service.Spec.Type) return false }
func validateLoadBalancer(t *testing.T, loadBalancer network.LoadBalancer, services ...api.Service) { expectedRuleCount := 0 for _, svc := range services { for _, wantedRule := range svc.Spec.Ports { expectedRuleCount++ wantedRuleName := getRuleName(&svc, wantedRule) foundRule := false for _, actualRule := range *loadBalancer.Properties.LoadBalancingRules { if strings.EqualFold(*actualRule.Name, wantedRuleName) && *actualRule.Properties.FrontendPort == wantedRule.Port && *actualRule.Properties.BackendPort == wantedRule.Port { foundRule = true break } } if !foundRule { t.Errorf("Expected load balancer rule but didn't find it: %q", wantedRuleName) } foundProbe := false if serviceapi.NeedsHealthCheck(&svc) { path, port := serviceapi.GetServiceHealthCheckPathPort(&svc) for _, actualProbe := range *loadBalancer.Properties.Probes { if strings.EqualFold(*actualProbe.Name, wantedRuleName) && *actualProbe.Properties.Port == port && *actualProbe.Properties.RequestPath == path && actualProbe.Properties.Protocol == network.ProbeProtocolHTTP { foundProbe = true break } } } else { for _, actualProbe := range *loadBalancer.Properties.Probes { if strings.EqualFold(*actualProbe.Name, wantedRuleName) && *actualProbe.Properties.Port == wantedRule.NodePort { foundProbe = true break } } } if !foundProbe { for _, actualProbe := range *loadBalancer.Properties.Probes { t.Logf("Probe: %s %d", *actualProbe.Name, *actualProbe.Properties.Port) } t.Errorf("Expected loadbalancer probe but didn't find it: %q", wantedRuleName) } } } lenRules := len(*loadBalancer.Properties.LoadBalancingRules) if lenRules != expectedRuleCount { t.Errorf("Expected the loadbalancer to have %d rules. Found %d.", expectedRuleCount, lenRules) } lenProbes := len(*loadBalancer.Properties.Probes) if lenProbes != expectedRuleCount { t.Errorf("Expected the loadbalancer to have %d probes. Found %d.", expectedRuleCount, lenProbes) } }
// OnServiceUpdate tracks the active set of service proxies. // They will be synchronized using syncProxyRules() func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) { start := time.Now() defer func() { glog.V(4).Infof("OnServiceUpdate took %v for %d services", time.Since(start), len(allServices)) }() proxier.mu.Lock() defer proxier.mu.Unlock() proxier.haveReceivedServiceUpdate = true activeServices := make(map[proxy.ServicePortName]bool) // use a map as a set for i := range allServices { service := &allServices[i] svcName := types.NamespacedName{ Namespace: service.Namespace, Name: service.Name, } // if ClusterIP is "None" or empty, skip proxying if !api.IsServiceIPSet(service) { glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) continue } for i := range service.Spec.Ports { servicePort := &service.Spec.Ports[i] serviceName := proxy.ServicePortName{ NamespacedName: svcName, Port: servicePort.Name, } activeServices[serviceName] = true info, exists := proxier.serviceMap[serviceName] if exists && proxier.sameConfig(info, service, servicePort) { // Nothing changed. continue } if exists { // Something changed. glog.V(3).Infof("Something changed for service %q: removing it", serviceName) delete(proxier.serviceMap, serviceName) } serviceIP := net.ParseIP(service.Spec.ClusterIP) glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, serviceIP, servicePort.Port, servicePort.Protocol) info = newServiceInfo(serviceName) info.clusterIP = serviceIP info.port = int(servicePort.Port) info.protocol = servicePort.Protocol info.nodePort = int(servicePort.NodePort) info.externalIPs = service.Spec.ExternalIPs // Deep-copy in case the service instance changes info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer) info.sessionAffinityType = service.Spec.SessionAffinity info.loadBalancerSourceRanges = service.Spec.LoadBalancerSourceRanges info.onlyNodeLocalEndpoints = apiservice.NeedsHealthCheck(service) && featuregate.DefaultFeatureGate.ExternalTrafficLocalOnly() if info.onlyNodeLocalEndpoints { p := apiservice.GetServiceHealthCheckNodePort(service) if p == 0 { glog.Errorf("Service does not contain necessary annotation %v", apiservice.AnnotationHealthCheckNodePort) } else { info.healthCheckNodePort = int(p) // Turn on healthcheck responder to listen on the health check nodePort healthcheck.AddServiceListener(serviceName.NamespacedName, info.healthCheckNodePort) } } proxier.serviceMap[serviceName] = info glog.V(4).Infof("added serviceInfo(%s): %s", serviceName, spew.Sdump(info)) } } staleUDPServices := sets.NewString() // Remove serviceports missing from the update. for name, info := range proxier.serviceMap { if !activeServices[name] { glog.V(1).Infof("Removing service %q", name) if info.protocol == api.ProtocolUDP { staleUDPServices.Insert(info.clusterIP.String()) } delete(proxier.serviceMap, name) if info.onlyNodeLocalEndpoints && info.healthCheckNodePort > 0 { // Remove ServiceListener health check nodePorts from the health checker // TODO - Stats healthcheck.DeleteServiceListener(name.NamespacedName, info.healthCheckNodePort) } } } proxier.syncProxyRules() proxier.deleteServiceConnections(staleUDPServices.List()) }
// This ensures load balancer exists and the frontend ip config is setup. // This also reconciles the Service's Ports with the LoadBalancer config. // This entails adding rules/probes for expected Ports and removing stale rules/ports. func (az *Cloud) reconcileLoadBalancer(lb network.LoadBalancer, pip *network.PublicIPAddress, clusterName string, service *api.Service, nodeNames []string) (network.LoadBalancer, bool, error) { lbName := getLoadBalancerName(clusterName) serviceName := getServiceName(service) lbFrontendIPConfigName := getFrontendIPConfigName(service) lbFrontendIPConfigID := az.getFrontendIPConfigID(lbName, lbFrontendIPConfigName) lbBackendPoolName := getBackendPoolName(clusterName) lbBackendPoolID := az.getBackendPoolID(lbName, lbBackendPoolName) wantLb := len(service.Spec.Ports) > 0 dirtyLb := false // Ensure LoadBalancer's Backend Pool Configuration if wantLb { if lb.Properties.BackendAddressPools == nil || len(*lb.Properties.BackendAddressPools) == 0 { lb.Properties.BackendAddressPools = &[]network.BackendAddressPool{ { Name: to.StringPtr(lbBackendPoolName), }, } glog.V(10).Infof("reconcile(%s)(%t): lb backendpool - adding", serviceName, wantLb) dirtyLb = true } else if len(*lb.Properties.BackendAddressPools) != 1 || !strings.EqualFold(*(*lb.Properties.BackendAddressPools)[0].Name, lbBackendPoolName) { return lb, false, fmt.Errorf("loadbalancer is misconfigured with a different backend pool") } } // Ensure LoadBalancer's Frontend IP Configurations dirtyConfigs := false newConfigs := []network.FrontendIPConfiguration{} if lb.Properties.FrontendIPConfigurations != nil { newConfigs = *lb.Properties.FrontendIPConfigurations } if !wantLb { for i := len(newConfigs) - 1; i >= 0; i-- { config := newConfigs[i] if strings.EqualFold(*config.Name, lbFrontendIPConfigName) { glog.V(3).Infof("reconcile(%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, lbFrontendIPConfigName) newConfigs = append(newConfigs[:i], newConfigs[i+1:]...) dirtyConfigs = true } } } else { foundConfig := false for _, config := range newConfigs { if strings.EqualFold(*config.Name, lbFrontendIPConfigName) { foundConfig = true break } } if !foundConfig { newConfigs = append(newConfigs, network.FrontendIPConfiguration{ Name: to.StringPtr(lbFrontendIPConfigName), Properties: &network.FrontendIPConfigurationPropertiesFormat{ PublicIPAddress: &network.PublicIPAddress{ ID: pip.ID, }, }, }) glog.V(10).Infof("reconcile(%s)(%t): lb frontendconfig(%s) - adding", serviceName, wantLb, lbFrontendIPConfigName) dirtyConfigs = true } } if dirtyConfigs { dirtyLb = true lb.Properties.FrontendIPConfigurations = &newConfigs } // update probes/rules expectedProbes := make([]network.Probe, len(service.Spec.Ports)) expectedRules := make([]network.LoadBalancingRule, len(service.Spec.Ports)) for i, port := range service.Spec.Ports { lbRuleName := getRuleName(service, port) transportProto, _, probeProto, err := getProtocolsFromKubernetesProtocol(port.Protocol) if err != nil { return lb, false, err } if serviceapi.NeedsHealthCheck(service) { podPresencePath, podPresencePort := serviceapi.GetServiceHealthCheckPathPort(service) expectedProbes[i] = network.Probe{ Name: &lbRuleName, Properties: &network.ProbePropertiesFormat{ RequestPath: to.StringPtr(podPresencePath), Protocol: network.ProbeProtocolHTTP, Port: to.Int32Ptr(podPresencePort), IntervalInSeconds: to.Int32Ptr(5), NumberOfProbes: to.Int32Ptr(2), }, } } else { expectedProbes[i] = network.Probe{ Name: &lbRuleName, Properties: &network.ProbePropertiesFormat{ Protocol: probeProto, Port: to.Int32Ptr(port.NodePort), IntervalInSeconds: to.Int32Ptr(5), NumberOfProbes: to.Int32Ptr(2), }, } } expectedRules[i] = network.LoadBalancingRule{ Name: &lbRuleName, Properties: &network.LoadBalancingRulePropertiesFormat{ Protocol: transportProto, FrontendIPConfiguration: &network.SubResource{ ID: to.StringPtr(lbFrontendIPConfigID), }, BackendAddressPool: &network.SubResource{ ID: to.StringPtr(lbBackendPoolID), }, Probe: &network.SubResource{ ID: to.StringPtr(az.getLoadBalancerProbeID(lbName, lbRuleName)), }, FrontendPort: to.Int32Ptr(port.Port), BackendPort: to.Int32Ptr(port.Port), EnableFloatingIP: to.BoolPtr(true), }, } } // remove unwanted probes dirtyProbes := false var updatedProbes []network.Probe if lb.Properties.Probes != nil { updatedProbes = *lb.Properties.Probes } for i := len(updatedProbes) - 1; i >= 0; i-- { existingProbe := updatedProbes[i] if serviceOwnsRule(service, *existingProbe.Name) { glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - considering evicting", serviceName, wantLb, *existingProbe.Name) keepProbe := false if findProbe(expectedProbes, existingProbe) { glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - keeping", serviceName, wantLb, *existingProbe.Name) keepProbe = true } if !keepProbe { updatedProbes = append(updatedProbes[:i], updatedProbes[i+1:]...) glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - dropping", serviceName, wantLb, *existingProbe.Name) dirtyProbes = true } } } // add missing, wanted probes for _, expectedProbe := range expectedProbes { foundProbe := false if findProbe(updatedProbes, expectedProbe) { glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - already exists", serviceName, wantLb, *expectedProbe.Name) foundProbe = true } if !foundProbe { glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - adding", serviceName, wantLb, *expectedProbe.Name) updatedProbes = append(updatedProbes, expectedProbe) dirtyProbes = true } } if dirtyProbes { dirtyLb = true lb.Properties.Probes = &updatedProbes } // update rules dirtyRules := false var updatedRules []network.LoadBalancingRule if lb.Properties.LoadBalancingRules != nil { updatedRules = *lb.Properties.LoadBalancingRules } // update rules: remove unwanted for i := len(updatedRules) - 1; i >= 0; i-- { existingRule := updatedRules[i] if serviceOwnsRule(service, *existingRule.Name) { keepRule := false glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name) if findRule(expectedRules, existingRule) { glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - keeping", serviceName, wantLb, *existingRule.Name) keepRule = true } if !keepRule { glog.V(3).Infof("reconcile(%s)(%t): lb rule(%s) - dropping", serviceName, wantLb, *existingRule.Name) updatedRules = append(updatedRules[:i], updatedRules[i+1:]...) dirtyRules = true } } } // update rules: add needed for _, expectedRule := range expectedRules { foundRule := false if findRule(updatedRules, expectedRule) { glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name) foundRule = true } if !foundRule { glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) adding", serviceName, wantLb, *expectedRule.Name) updatedRules = append(updatedRules, expectedRule) dirtyRules = true } } if dirtyRules { dirtyLb = true lb.Properties.LoadBalancingRules = &updatedRules } return lb, dirtyLb, nil }