// Validate allocation of a nodePort when the externalTraffic=OnlyLocal annotation is set
// and type is LoadBalancer
func TestServiceRegistryExternalTrafficAnnotationHealthCheckNodePortAllocation(t *testing.T) {
ctx := genericapirequest.NewDefaultContext()
storage, _ := NewTestREST(t, nil)
svc := &api.Service{
ObjectMeta: metav1.ObjectMeta{Name: "external-lb-esipp",
Annotations: map[string]string{
service.BetaAnnotationExternalTraffic: service.AnnotationValueExternalTrafficLocal,
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{"bar": "baz"},
SessionAffinity: api.ServiceAffinityNone,
Type: api.ServiceTypeLoadBalancer,
Ports: []api.ServicePort{{
Port: 6502,
Protocol: api.ProtocolTCP,
TargetPort: intstr.FromInt(6502),
}},
},
}
created_svc, err := storage.Create(ctx, svc)
if created_svc == nil || err != nil {
t.Errorf("Unexpected failure creating service %v", err)
}
created_service := created_svc.(*api.Service)
if !service.NeedsHealthCheck(created_service) {
t.Errorf("Unexpected missing annotation %s", service.BetaAnnotationExternalTraffic)
}
port := service.GetServiceHealthCheckNodePort(created_service)
if port == 0 {
t.Errorf("Failed to allocate and create the health check node port annotation %s", service.BetaAnnotationHealthCheckNodePort)
}
}
// Validate that the health check nodePort is not allocated when service type is ClusterIP
func TestServiceRegistryExternalTrafficAnnotationClusterIP(t *testing.T) {
ctx := api.NewDefaultContext()
storage, _ := NewTestREST(t, nil)
svc := &api.Service{
ObjectMeta: api.ObjectMeta{Name: "external-lb-esipp",
Annotations: map[string]string{
service.AnnotationExternalTraffic: service.AnnotationValueExternalTrafficGlobal,
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{"bar": "baz"},
SessionAffinity: api.ServiceAffinityNone,
Type: api.ServiceTypeClusterIP,
Ports: []api.ServicePort{{
Port: 6502,
Protocol: api.ProtocolTCP,
TargetPort: intstr.FromInt(6502),
}},
},
}
created_svc, err := storage.Create(ctx, svc)
if created_svc == nil || err != nil {
t.Errorf("Unexpected failure creating service %v", err)
}
created_service := created_svc.(*api.Service)
// Make sure that ClusterIP services do not have the health check node port allocated
port := service.GetServiceHealthCheckNodePort(created_service)
if port != 0 {
t.Errorf("Unexpected allocation of health check node port annotation %s", service.AnnotationHealthCheckNodePort)
}
}
func (rs *REST) Delete(ctx api.Context, id string) (runtime.Object, error) {
service, err := rs.registry.GetService(ctx, id)
if err != nil {
return nil, err
}
err = rs.registry.DeleteService(ctx, id)
if err != nil {
return nil, err
}
// TODO: can leave dangling endpoints, and potentially return incorrect
// endpoints if a new service is created with the same name
err = rs.endpoints.DeleteEndpoints(ctx, id)
if err != nil && !errors.IsNotFound(err) {
return nil, err
}
if api.IsServiceIPSet(service) {
rs.serviceIPs.Release(net.ParseIP(service.Spec.ClusterIP))
}
for _, nodePort := range CollectServiceNodePorts(service) {
err := rs.serviceNodePorts.Release(nodePort)
if err != nil {
// these should be caught by an eventual reconciliation / restart
glog.Errorf("Error releasing service %s node port %d: %v", service.Name, nodePort, err)
}
}
if shouldCheckOrAssignHealthCheckNodePort(service) {
nodePort := apiservice.GetServiceHealthCheckNodePort(service)
if nodePort > 0 {
err := rs.serviceNodePorts.Release(int(nodePort))
if err != nil {
// these should be caught by an eventual reconciliation / restart
utilruntime.HandleError(fmt.Errorf("Error releasing service health check %s node port %d: %v", service.Name, nodePort, err))
}
}
}
return &unversioned.Status{Status: unversioned.StatusSuccess}, nil
}
func (rs *REST) healthCheckNodePortUpdate(oldService, service *api.Service) (bool, error) {
// Health Check Node Port handling during updates
//
// Case 1. Transition from globalTraffic to OnlyLocal for the ESIPP annotation
//
// Allocate a health check node port or attempt to reserve the user-specified one, if provided.
// Insert health check node port as an annotation into the service's annotations
//
// Case 2. Transition from OnlyLocal to Global for the ESIPP annotation
//
// Free the existing healthCheckNodePort and clear the health check nodePort annotation
//
// Case 3. No change (Global ---stays--> Global) but prevent invalid annotation manipulations
//
// Reject insertion of the "service.alpha.kubernetes.io/healthcheck-nodeport" annotation
//
// Case 4. No change (OnlyLocal ---stays--> OnlyLocal) but prevent invalid annotation manipulations
//
// Reject deletion of the "service.alpha.kubernetes.io/healthcheck-nodeport" annotation
// Reject changing the value of the healthCheckNodePort annotation
//
oldServiceHasHealthCheckNodePort := shouldCheckOrAssignHealthCheckNodePort(oldService)
oldHealthCheckNodePort := apiservice.GetServiceHealthCheckNodePort(oldService)
assignHealthCheckNodePort := shouldCheckOrAssignHealthCheckNodePort(service)
requestedHealthCheckNodePort := apiservice.GetServiceHealthCheckNodePort(service)
switch {
case !oldServiceHasHealthCheckNodePort && assignHealthCheckNodePort:
glog.Infof("Transition from Global LB service to OnlyLocal service")
if requestedHealthCheckNodePort > 0 {
// If the request has a health check nodePort in mind, attempt to reserve it
err := rs.serviceNodePorts.Allocate(int(requestedHealthCheckNodePort))
if err != nil {
errmsg := fmt.Sprintf("Failed to allocate requested HealthCheck nodePort %v:%v",
requestedHealthCheckNodePort, err)
el := field.ErrorList{field.Invalid(field.NewPath("metadata", "annotations"),
apiservice.AnnotationHealthCheckNodePort, errmsg)}
return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
}
glog.Infof("Reserved user requested nodePort: %d", requestedHealthCheckNodePort)
} else {
// If the request has no health check nodePort specified, allocate any
healthCheckNodePort, err := rs.serviceNodePorts.AllocateNext()
if err != nil {
// TODO: what error should be returned here? It's not a
// field-level validation failure (the field is valid), and it's
// not really an internal error.
return false, errors.NewInternalError(fmt.Errorf("failed to allocate a nodePort: %v", err))
}
// Insert the newly allocated health check port as an annotation (plan of record for Alpha)
service.Annotations[apiservice.AnnotationHealthCheckNodePort] = fmt.Sprintf("%d", healthCheckNodePort)
glog.Infof("Reserved health check nodePort: %d", healthCheckNodePort)
}
case oldServiceHasHealthCheckNodePort && !assignHealthCheckNodePort:
glog.Infof("Transition from OnlyLocal LB service to Global service")
err := rs.serviceNodePorts.Release(int(oldHealthCheckNodePort))
if err != nil {
glog.Warningf("Error releasing service health check %s node port %d: %v", service.Name, oldHealthCheckNodePort, err)
return false, errors.NewInternalError(fmt.Errorf("failed to free health check nodePort: %v", err))
} else {
delete(service.Annotations, apiservice.AnnotationHealthCheckNodePort)
glog.Infof("Freed health check nodePort: %d", oldHealthCheckNodePort)
}
case !oldServiceHasHealthCheckNodePort && !assignHealthCheckNodePort:
if _, ok := service.Annotations[apiservice.AnnotationHealthCheckNodePort]; ok {
glog.Warningf("Attempt to insert health check node port annotation DENIED")
el := field.ErrorList{field.Invalid(field.NewPath("metadata", "annotations"),
apiservice.AnnotationHealthCheckNodePort, "Cannot insert healthcheck nodePort annotation")}
return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
}
case oldServiceHasHealthCheckNodePort && assignHealthCheckNodePort:
if _, ok := service.Annotations[apiservice.AnnotationHealthCheckNodePort]; !ok {
glog.Warningf("Attempt to delete health check node port annotation DENIED")
el := field.ErrorList{field.Invalid(field.NewPath("metadata", "annotations"),
apiservice.AnnotationHealthCheckNodePort, "Cannot delete healthcheck nodePort annotation")}
return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
}
if oldHealthCheckNodePort != requestedHealthCheckNodePort {
glog.Warningf("Attempt to change value of health check node port annotation DENIED")
el := field.ErrorList{field.Invalid(field.NewPath("metadata", "annotations"),
apiservice.AnnotationHealthCheckNodePort, "Cannot change healthcheck nodePort during update")}
return false, errors.NewInvalid(api.Kind("Service"), service.Name, el)
}
}
return true, nil
}
// OnServiceUpdate tracks the active set of service proxies.
// They will be synchronized using syncProxyRules()
func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) {
start := time.Now()
defer func() {
glog.V(4).Infof("OnServiceUpdate took %v for %d services", time.Since(start), len(allServices))
}()
proxier.mu.Lock()
defer proxier.mu.Unlock()
proxier.haveReceivedServiceUpdate = true
activeServices := make(map[proxy.ServicePortName]bool) // use a map as a set
for i := range allServices {
service := &allServices[i]
svcName := types.NamespacedName{
Namespace: service.Namespace,
Name: service.Name,
}
// if ClusterIP is "None" or empty, skip proxying
if !api.IsServiceIPSet(service) {
glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP)
continue
}
for i := range service.Spec.Ports {
servicePort := &service.Spec.Ports[i]
serviceName := proxy.ServicePortName{
NamespacedName: svcName,
Port: servicePort.Name,
}
activeServices[serviceName] = true
info, exists := proxier.serviceMap[serviceName]
if exists && proxier.sameConfig(info, service, servicePort) {
// Nothing changed.
continue
}
if exists {
// Something changed.
glog.V(3).Infof("Something changed for service %q: removing it", serviceName)
delete(proxier.serviceMap, serviceName)
}
serviceIP := net.ParseIP(service.Spec.ClusterIP)
glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, serviceIP, servicePort.Port, servicePort.Protocol)
info = newServiceInfo(serviceName)
info.clusterIP = serviceIP
info.port = int(servicePort.Port)
info.protocol = servicePort.Protocol
info.nodePort = int(servicePort.NodePort)
info.externalIPs = service.Spec.ExternalIPs
// Deep-copy in case the service instance changes
info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer)
info.sessionAffinityType = service.Spec.SessionAffinity
info.loadBalancerSourceRanges = service.Spec.LoadBalancerSourceRanges
info.onlyNodeLocalEndpoints = apiservice.NeedsHealthCheck(service) && featuregate.DefaultFeatureGate.ExternalTrafficLocalOnly()
if info.onlyNodeLocalEndpoints {
p := apiservice.GetServiceHealthCheckNodePort(service)
if p == 0 {
glog.Errorf("Service does not contain necessary annotation %v",
apiservice.AnnotationHealthCheckNodePort)
} else {
info.healthCheckNodePort = int(p)
// Turn on healthcheck responder to listen on the health check nodePort
healthcheck.AddServiceListener(serviceName.NamespacedName, info.healthCheckNodePort)
}
}
proxier.serviceMap[serviceName] = info
glog.V(4).Infof("added serviceInfo(%s): %s", serviceName, spew.Sdump(info))
}
}
staleUDPServices := sets.NewString()
// Remove serviceports missing from the update.
for name, info := range proxier.serviceMap {
if !activeServices[name] {
glog.V(1).Infof("Removing service %q", name)
if info.protocol == api.ProtocolUDP {
staleUDPServices.Insert(info.clusterIP.String())
}
delete(proxier.serviceMap, name)
if info.onlyNodeLocalEndpoints && info.healthCheckNodePort > 0 {
// Remove ServiceListener health check nodePorts from the health checker
// TODO - Stats
healthcheck.DeleteServiceListener(name.NamespacedName, info.healthCheckNodePort)
}
}
}
proxier.syncProxyRules()
proxier.deleteServiceConnections(staleUDPServices.List())
}