func addDNS(record string, service *kapi.Service, etcdClient *etcd.Client) error {
// if PortalIP is not set, a DNS entry should not be created
if !kapi.IsServiceIPSet(service) {
log.Printf("Skipping dns record for headless service: %s\n", service.Name)
return nil
}
for i := range service.Spec.Ports {
svc := skymsg.Service{
Host: service.Spec.PortalIP,
Port: service.Spec.Ports[i].Port,
Priority: 10,
Weight: 10,
Ttl: 30,
}
b, err := json.Marshal(svc)
if err != nil {
return err
}
// Set with no TTL, and hope that kubernetes events are accurate.
log.Printf("Setting DNS record: %v -> %s:%d\n", record, service.Spec.PortalIP, service.Spec.Ports[i].Port)
_, err = etcdClient.Set(skymsg.Path(record), string(b), uint64(0))
if err != nil {
return err
}
}
return nil
}
func (ks *kube2consul) addDNS(record string, service *kapi.Service) error {
if strings.Contains(record, ".") {
glog.V(1).Infof("Service names containing '.' are not supported: %s\n", service.Name)
return nil
}
// if PortalIP is not set, do not create a DNS records
if !kapi.IsServiceIPSet(service) {
glog.V(1).Infof("Skipping dns record for headless service: %s\n", service.Name)
return nil
}
for i := range service.Spec.Ports {
asr := &consulapi.AgentServiceRegistration{
ID: record,
Name: record,
Address: service.Spec.PortalIP,
Port: service.Spec.Ports[0].Port,
}
glog.V(2).Infof("Setting DNS record: %v -> %v:%d\n", record, service.Spec.PortalIP, service.Spec.Ports[i].Port)
if err := ks.consulClient.Agent().ServiceRegister(asr); err != nil {
return err
}
}
return nil
}
// FromServices builds environment variables that a container is started with,
// which tell the container where to find the services it may need, which are
// provided as an argument.
func FromServices(services *api.ServiceList) []api.EnvVar {
var result []api.EnvVar
for i := range services.Items {
service := &services.Items[i]
// ignore services where PortalIP is "None" or empty
// the services passed to this method should be pre-filtered
// only services that have the portal IP set should be included here
if !api.IsServiceIPSet(service) {
continue
}
// Host
name := makeEnvVariableName(service.Name) + "_SERVICE_HOST"
result = append(result, api.EnvVar{Name: name, Value: service.Spec.PortalIP})
// First port - give it the backwards-compatible name
name = makeEnvVariableName(service.Name) + "_SERVICE_PORT"
result = append(result, api.EnvVar{Name: name, Value: strconv.Itoa(service.Spec.Ports[0].Port)})
// All named ports (only the first may be unnamed, checked in validation)
for i := range service.Spec.Ports {
sp := &service.Spec.Ports[i]
if sp.Name != "" {
pn := name + "_" + makeEnvVariableName(sp.Name)
result = append(result, api.EnvVar{Name: pn, Value: strconv.Itoa(sp.Port)})
}
}
// Docker-compatible vars.
result = append(result, makeLinkVariables(service)...)
}
return result
}
func (rs *REST) Delete(ctx api.Context, id string) (runtime.Object, error) {
service, err := rs.registry.GetService(ctx, id)
if err != nil {
return nil, err
}
err = rs.registry.DeleteService(ctx, id)
if err != nil {
return nil, err
}
if api.IsServiceIPSet(service) {
rs.serviceIPs.Release(net.ParseIP(service.Spec.ClusterIP))
}
for _, nodePort := range CollectServiceNodePorts(service) {
err := rs.serviceNodePorts.Release(nodePort)
if err != nil {
// these should be caught by an eventual reconciliation / restart
glog.Errorf("Error releasing service %s node port %d: %v", service.Name, nodePort, err)
}
}
return &api.Status{Status: api.StatusSuccess}, nil
}
// RunOnce verifies the state of the portal IP allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) RunOnce() error {
// TODO: (per smarterclayton) if Get() or ListServices() is a weak consistency read,
// or if they are executed against different leaders,
// the ordering guarantee required to ensure no IP is allocated twice is violated.
// ListServices must return a ResourceVersion higher than the etcd index Get triggers,
// and the release code must not release services that have had IPs allocated but not yet been created
// See #8295
latest, err := c.alloc.Get()
if err != nil {
return fmt.Errorf("unable to refresh the service IP block: %v", err)
}
ctx := api.WithNamespace(api.NewDefaultContext(), api.NamespaceAll)
list, err := c.registry.ListServices(ctx)
if err != nil {
return fmt.Errorf("unable to refresh the service IP block: %v", err)
}
r := ipallocator.NewCIDRRange(c.network)
for _, svc := range list.Items {
if !api.IsServiceIPSet(&svc) {
continue
}
ip := net.ParseIP(svc.Spec.PortalIP)
if ip == nil {
// portal IP is broken, reallocate
util.HandleError(fmt.Errorf("the portal IP %s for service %s/%s is not a valid IP; please recreate", svc.Spec.PortalIP, svc.Name, svc.Namespace))
continue
}
switch err := r.Allocate(ip); err {
case nil:
case ipallocator.ErrAllocated:
// TODO: send event
// portal IP is broken, reallocate
util.HandleError(fmt.Errorf("the portal IP %s for service %s/%s was assigned to multiple services; please recreate", ip, svc.Name, svc.Namespace))
case ipallocator.ErrNotInRange:
// TODO: send event
// portal IP is broken, reallocate
util.HandleError(fmt.Errorf("the portal IP %s for service %s/%s is not within the service CIDR %s; please recreate", ip, svc.Name, svc.Namespace, c.network))
case ipallocator.ErrFull:
// TODO: send event
return fmt.Errorf("the service CIDR %s is full; you must widen the CIDR in order to create new services")
default:
return fmt.Errorf("unable to allocate portal IP %s for service %s/%s due to an unknown error, exiting: %v", ip, svc.Name, svc.Namespace, err)
}
}
err = r.Snapshot(latest)
if err != nil {
return fmt.Errorf("unable to persist the updated service IP allocations: %v", err)
}
if err := c.alloc.CreateOrUpdate(latest); err != nil {
return fmt.Errorf("unable to persist the updated service IP allocations: %v", err)
}
return nil
}
func (rs *REST) Create(ctx api.Context, obj runtime.Object) (runtime.Object, error) {
service := obj.(*api.Service)
if err := rest.BeforeCreate(rest.Services, ctx, obj); err != nil {
return nil, err
}
releaseServiceIP := false
defer func() {
if releaseServiceIP {
if api.IsServiceIPSet(service) {
rs.portalMgr.Release(net.ParseIP(service.Spec.PortalIP))
}
}
}()
if api.IsServiceIPRequested(service) {
// Allocate next available.
ip, err := rs.portalMgr.AllocateNext()
if err != nil {
return nil, err
}
service.Spec.PortalIP = ip.String()
releaseServiceIP = true
} else if api.IsServiceIPSet(service) {
// Try to respect the requested IP.
if err := rs.portalMgr.Allocate(net.ParseIP(service.Spec.PortalIP)); err != nil {
el := fielderrors.ValidationErrorList{fielderrors.NewFieldInvalid("spec.portalIP", service.Spec.PortalIP, err.Error())}
return nil, errors.NewInvalid("Service", service.Name, el)
}
releaseServiceIP = true
}
out, err := rs.registry.CreateService(ctx, service)
if err != nil {
err = rest.CheckGeneratedNameError(rest.Services, err, service)
}
if err == nil {
releaseServiceIP = false
}
return out, err
}
func (ks *kube2sky) addDNS(subdomain string, service *kapi.Service, isNewStyleFormat bool) error {
if len(service.Spec.Ports) == 0 {
glog.Fatalf("unexpected service with no ports: %v", service)
}
// if ClusterIP is not set, a DNS entry should not be created
if !kapi.IsServiceIPSet(service) {
return ks.newHeadlessService(subdomain, service, isNewStyleFormat)
}
return ks.generateRecordsForPortalService(subdomain, service, isNewStyleFormat)
}
func (rs *REST) Delete(ctx api.Context, id string) (runtime.Object, error) {
service, err := rs.registry.GetService(ctx, id)
if err != nil {
return nil, err
}
if api.IsServiceIPSet(service) {
rs.portalMgr.Release(net.ParseIP(service.Spec.PortalIP))
}
return &api.Status{Status: api.StatusSuccess}, rs.registry.DeleteService(ctx, id)
}
// ValidateServiceUpdate tests if required fields in the service are set during an update
func ValidateServiceUpdate(oldService, service *api.Service) errs.ValidationErrorList {
allErrs := errs.ValidationErrorList{}
allErrs = append(allErrs, ValidateObjectMetaUpdate(&oldService.ObjectMeta, &service.ObjectMeta).Prefix("metadata")...)
// TODO: PortalIP should be a Status field, since the system can set a value != to the user's value
// once PortalIP is set, it cannot be unset.
if api.IsServiceIPSet(oldService) && service.Spec.PortalIP != oldService.Spec.PortalIP {
allErrs = append(allErrs, errs.NewFieldInvalid("spec.portalIP", service.Spec.PortalIP, "field is immutable"))
}
return allErrs
}
func (ks *kube2sky) addDNSUsingEndpoints(subdomain string, e *kapi.Endpoints, isNewStyleFormat bool) error {
ks.mlock.Lock()
defer ks.mlock.Unlock()
svc, err := ks.getServiceFromEndpoints(e)
if err != nil {
return err
}
if svc == nil || kapi.IsServiceIPSet(svc) {
// No headless service found corresponding to endpoints object.
return nil
}
// Remove existing DNS entry.
if err := ks.removeDNS(subdomain); err != nil {
return err
}
return ks.generateRecordsForHeadlessService(subdomain, e, svc, isNewStyleFormat)
}
// Helper: mark all previously allocated IPs in the allocator.
func reloadIPsFromStorage(ipa *ipAllocator, registry Registry) {
services, err := registry.ListServices(api.NewContext())
if err != nil {
// This is really bad.
glog.Errorf("can't list services to init service REST: %v", err)
return
}
for i := range services.Items {
service := &services.Items[i]
if !api.IsServiceIPSet(service) {
continue
}
if err := ipa.Allocate(net.ParseIP(service.Spec.PortalIP)); err != nil {
// This is really bad.
glog.Errorf("service %q PortalIP %s could not be allocated: %v", service.Name, service.Spec.PortalIP, err)
}
}
}
// ValidateService tests if required fields in the service are set.
func ValidateService(service *api.Service) errs.ValidationErrorList {
allErrs := errs.ValidationErrorList{}
allErrs = append(allErrs, ValidateObjectMeta(&service.ObjectMeta, true, ValidateServiceName).Prefix("metadata")...)
if !util.IsValidPortNum(service.Spec.Port) {
allErrs = append(allErrs, errs.NewFieldInvalid("spec.port", service.Spec.Port, portRangeErrorMsg))
}
if len(service.Spec.Protocol) == 0 {
allErrs = append(allErrs, errs.NewFieldRequired("spec.protocol"))
} else if !supportedPortProtocols.Has(strings.ToUpper(string(service.Spec.Protocol))) {
allErrs = append(allErrs, errs.NewFieldNotSupported("spec.protocol", service.Spec.Protocol))
}
if service.Spec.TargetPort.Kind == util.IntstrInt && service.Spec.TargetPort.IntVal != 0 && !util.IsValidPortNum(service.Spec.TargetPort.IntVal) {
allErrs = append(allErrs, errs.NewFieldInvalid("spec.containerPort", service.Spec.Port, portRangeErrorMsg))
} else if service.Spec.TargetPort.Kind == util.IntstrString && len(service.Spec.TargetPort.StrVal) == 0 {
allErrs = append(allErrs, errs.NewFieldRequired("spec.containerPort"))
}
if service.Spec.Selector != nil {
allErrs = append(allErrs, ValidateLabels(service.Spec.Selector, "spec.selector")...)
}
if service.Spec.SessionAffinity == "" {
allErrs = append(allErrs, errs.NewFieldRequired("spec.sessionAffinity"))
} else if !supportedSessionAffinityType.Has(string(service.Spec.SessionAffinity)) {
allErrs = append(allErrs, errs.NewFieldNotSupported("spec.sessionAffinity", service.Spec.SessionAffinity))
}
if api.IsServiceIPSet(service) {
if ip := net.ParseIP(service.Spec.PortalIP); ip == nil {
allErrs = append(allErrs, errs.NewFieldInvalid("spec.portalIP", service.Spec.PortalIP, "portalIP should be empty, 'None', or a valid IP address"))
}
}
return allErrs
}
// OnUpdate manages the active set of service proxies.
// Active service proxies are reinitialized if found in the update set or
// shutdown if missing from the update set.
func (proxier *Proxier) OnUpdate(services []api.Service) {
glog.V(4).Infof("Received update notice: %+v", services)
activeServices := make(map[ServicePortName]bool) // use a map as a set
for i := range services {
service := &services[i]
// if PortalIP is "None" or empty, skip proxying
if !api.IsServiceIPSet(service) {
glog.V(3).Infof("Skipping service %s due to portal IP = %q", types.NamespacedName{service.Namespace, service.Name}, service.Spec.PortalIP)
continue
}
for i := range service.Spec.Ports {
servicePort := &service.Spec.Ports[i]
serviceName := ServicePortName{types.NamespacedName{service.Namespace, service.Name}, servicePort.Name}
activeServices[serviceName] = true
serviceIP := net.ParseIP(service.Spec.PortalIP)
info, exists := proxier.getServiceInfo(serviceName)
// TODO: check health of the socket? What if ProxyLoop exited?
if exists && sameConfig(info, service, servicePort) {
// Nothing changed.
continue
}
if exists {
glog.V(4).Infof("Something changed for service %q: stopping it", serviceName)
err := proxier.closePortal(serviceName, info)
if err != nil {
glog.Errorf("Failed to close portal for %q: %v", serviceName, err)
}
err = proxier.stopProxy(serviceName, info)
if err != nil {
glog.Errorf("Failed to stop service %q: %v", serviceName, err)
}
}
glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, serviceIP, servicePort.Port, servicePort.Protocol)
info, err := proxier.addServiceOnPort(serviceName, servicePort.Protocol, 0, udpIdleTimeout)
if err != nil {
glog.Errorf("Failed to start proxy for %q: %v", serviceName, err)
continue
}
info.portalIP = serviceIP
info.portalPort = servicePort.Port
info.deprecatedPublicIPs = service.Spec.DeprecatedPublicIPs
// Deep-copy in case the service instance changes
info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer)
info.nodePort = servicePort.NodePort
info.sessionAffinityType = service.Spec.SessionAffinity
glog.V(4).Infof("info: %+v", info)
err = proxier.openPortal(serviceName, info)
if err != nil {
glog.Errorf("Failed to open portal for %q: %v", serviceName, err)
}
proxier.loadBalancer.NewService(serviceName, info.sessionAffinityType, info.stickyMaxAgeMinutes)
}
}
proxier.mu.Lock()
defer proxier.mu.Unlock()
for name, info := range proxier.serviceMap {
if !activeServices[name] {
glog.V(1).Infof("Stopping service %q", name)
err := proxier.closePortal(name, info)
if err != nil {
glog.Errorf("Failed to close portal for %q: %v", name, err)
}
err = proxier.stopProxyInternal(name, info)
if err != nil {
glog.Errorf("Failed to stop service %q: %v", name, err)
}
}
}
}
func (rs *REST) Create(ctx api.Context, obj runtime.Object) (runtime.Object, error) {
service := obj.(*api.Service)
if err := rest.BeforeCreate(rest.Services, ctx, obj); err != nil {
return nil, err
}
releaseServiceIP := false
defer func() {
if releaseServiceIP {
if api.IsServiceIPSet(service) {
rs.serviceIPs.Release(net.ParseIP(service.Spec.ClusterIP))
}
}
}()
nodePortOp := portallocator.StartOperation(rs.serviceNodePorts)
defer nodePortOp.Finish()
if api.IsServiceIPRequested(service) {
// Allocate next available.
ip, err := rs.serviceIPs.AllocateNext()
if err != nil {
el := fielderrors.ValidationErrorList{fielderrors.NewFieldInvalid("spec.clusterIP", service.Spec.ClusterIP, err.Error())}
return nil, errors.NewInvalid("Service", service.Name, el)
}
service.Spec.ClusterIP = ip.String()
releaseServiceIP = true
} else if api.IsServiceIPSet(service) {
// Try to respect the requested IP.
if err := rs.serviceIPs.Allocate(net.ParseIP(service.Spec.ClusterIP)); err != nil {
el := fielderrors.ValidationErrorList{fielderrors.NewFieldInvalid("spec.clusterIP", service.Spec.ClusterIP, err.Error())}
return nil, errors.NewInvalid("Service", service.Name, el)
}
releaseServiceIP = true
}
assignNodePorts := shouldAssignNodePorts(service)
for i := range service.Spec.Ports {
servicePort := &service.Spec.Ports[i]
if servicePort.NodePort != 0 {
err := nodePortOp.Allocate(servicePort.NodePort)
if err != nil {
el := fielderrors.ValidationErrorList{fielderrors.NewFieldInvalid("nodePort", servicePort.NodePort, err.Error())}.PrefixIndex(i).Prefix("spec.ports")
return nil, errors.NewInvalid("Service", service.Name, el)
}
} else if assignNodePorts {
nodePort, err := nodePortOp.AllocateNext()
if err != nil {
el := fielderrors.ValidationErrorList{fielderrors.NewFieldInvalid("nodePort", servicePort.NodePort, err.Error())}.PrefixIndex(i).Prefix("spec.ports")
return nil, errors.NewInvalid("Service", service.Name, el)
}
servicePort.NodePort = nodePort
}
}
out, err := rs.registry.CreateService(ctx, service)
if err != nil {
err = rest.CheckGeneratedNameError(rest.Services, err, service)
}
if err == nil {
el := nodePortOp.Commit()
if el != nil {
// these should be caught by an eventual reconciliation / restart
glog.Errorf("error(s) committing service node-ports changes: %v", el)
}
releaseServiceIP = false
}
return out, err
}