func (lbaas *LbaasV2) GetLoadBalancer(clusterName string, service *v1.Service) (*v1.LoadBalancerStatus, bool, error) { loadBalancerName := cloudprovider.GetLoadBalancerName(service) loadbalancer, err := getLoadbalancerByName(lbaas.network, loadBalancerName) if err == ErrNotFound { return nil, false, nil } if loadbalancer == nil { return nil, false, err } status := &v1.LoadBalancerStatus{} status.Ingress = []v1.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} return status, true, err }
func (lb *LoadBalancer) GetLoadBalancer(service *api.Service) (*api.LoadBalancerStatus, bool, error) { loadBalancerName := cloudprovider.GetLoadBalancerName(service) vip, err := getVipByName(lb.network, loadBalancerName) if err == ErrNotFound { return nil, false, nil } if vip == nil { return nil, false, err } status := &api.LoadBalancerStatus{} status.Ingress = []api.LoadBalancerIngress{{IP: vip.Address}} return status, true, err }
// Updates the external load balancer of a service, assuming we hold the mutex // associated with the service. func (s *ServiceController) lockedUpdateLoadBalancerHosts(service *api.Service, hosts []string) error { if !wantsExternalLoadBalancer(service) { return nil } name := cloudprovider.GetLoadBalancerName(service) err := s.balancer.UpdateTCPLoadBalancer(name, s.zone.Region, hosts) if err == nil { return nil } // It's only an actual error if the load balancer still exists. if _, exists, err := s.balancer.GetTCPLoadBalancer(name, s.zone.Region); err != nil { glog.Errorf("External error while checking if TCP load balancer %q exists: name, %v") } else if !exists { return nil } return err }
// ConstructFirewallForLBService returns the expected GCE firewall rule for a loadbalancer type service func ConstructFirewallForLBService(svc *v1.Service, nodesTags []string) *compute.Firewall { if svc.Spec.Type != v1.ServiceTypeLoadBalancer { Failf("can not construct firewall rule for non-loadbalancer type service") } fw := compute.Firewall{} fw.Name = MakeFirewallNameForLBService(cloudprovider.GetLoadBalancerName(svc)) fw.TargetTags = nodesTags if svc.Spec.LoadBalancerSourceRanges == nil { fw.SourceRanges = []string{"0.0.0.0/0"} } else { fw.SourceRanges = svc.Spec.LoadBalancerSourceRanges } for _, sp := range svc.Spec.Ports { fw.Allowed = append(fw.Allowed, &compute.FirewallAllowed{ IPProtocol: strings.ToLower(string(sp.Protocol)), Ports: []string{strconv.Itoa(int(sp.Port))}, }) } return &fw }
// Updates the load balancer of a service, assuming we hold the mutex // associated with the service. func (s *ServiceController) lockedUpdateLoadBalancerHosts(service *api.Service, hosts []string) error { if !wantsLoadBalancer(service) { return nil } // This operation doesn't normally take very long (and happens pretty often), so we only record the final event err := s.balancer.UpdateLoadBalancer(service, hosts) if err == nil { s.eventRecorder.Event(service, api.EventTypeNormal, "UpdatedLoadBalancer", "Updated load balancer with new hosts") return nil } // It's only an actual error if the load balancer still exists. if _, exists, err := s.balancer.GetLoadBalancer(service); err != nil { glog.Errorf("External error while checking if load balancer %q exists: name, %v", cloudprovider.GetLoadBalancerName(service), err) } else if !exists { return nil } s.eventRecorder.Eventf(service, api.EventTypeWarning, "LoadBalancerUpdateFailed", "Error updating load balancer with new hosts %v: %v", hosts, err) return err }
// EnsureLoadBalancer is a test-spy implementation of LoadBalancer.EnsureLoadBalancer. // It adds an entry "create" into the internal method call record. func (f *FakeCloud) EnsureLoadBalancer(service *api.Service, hosts []string) (*api.LoadBalancerStatus, error) { f.addCall("create") if f.Balancers == nil { f.Balancers = make(map[string]FakeBalancer) } name := cloudprovider.GetLoadBalancerName(service) spec := service.Spec zone, err := f.GetZone() if err != nil { return nil, err } region := zone.Region f.Balancers[name] = FakeBalancer{name, region, spec.LoadBalancerIP, spec.Ports, hosts} status := &api.LoadBalancerStatus{} status.Ingress = []api.LoadBalancerIngress{{IP: f.ExternalIP.String()}} return status, f.Err }
// getLoadBalancer retrieves the IP address and ID and all the existing rules it can find. func (cs *CSCloud) getLoadBalancer(service *api.Service) (*loadBalancer, error) { lb := &loadBalancer{ CloudStackClient: cs.client, name: cloudprovider.GetLoadBalancerName(service), projectID: cs.projectID, rules: make(map[string]*cloudstack.LoadBalancerRule), } p := cs.client.LoadBalancer.NewListLoadBalancerRulesParams() p.SetKeyword(lb.name) p.SetListall(true) if cs.projectID != "" { p.SetProjectid(cs.projectID) } l, err := cs.client.LoadBalancer.ListLoadBalancerRules(p) if err != nil { return nil, fmt.Errorf("error retrieving load balancer rules: %v", err) } for _, lbRule := range l.LoadBalancerRules { lb.rules[lbRule.Name] = lbRule if lb.ipAddr != "" && lb.ipAddr != lbRule.Publicip { glog.Warningf("Load balancer for service %v/%v has rules associated with different IP's: %v, %v", service.Namespace, service.Name, lb.ipAddr, lbRule.Publicip) } lb.ipAddr = lbRule.Publicip lb.ipAddrID = lbRule.Publicipid } glog.V(4).Infof("Load balancer %v contains %d rule(s)", lb.name, len(lb.rules)) return lb, nil }
// Updates the external load balancer of a service, assuming we hold the mutex // associated with the service. func (s *ServiceController) lockedUpdateLoadBalancerHosts(service *api.Service, hosts []string) error { if !wantsExternalLoadBalancer(service) { return nil } // This operation doesn't normally take very long (and happens pretty often), so we only record the final event name := cloudprovider.GetLoadBalancerName(service) err := s.balancer.UpdateTCPLoadBalancer(name, s.zone.Region, hosts) if err == nil { s.eventRecorder.Event(service, "updated loadbalancer", "updated loadbalancer with new hosts") return nil } // It's only an actual error if the load balancer still exists. if _, exists, err := s.balancer.GetTCPLoadBalancer(name, s.zone.Region); err != nil { glog.Errorf("External error while checking if TCP load balancer %q exists: name, %v", name, err) } else if !exists { return nil } message := "error updating loadbalancer with new hosts: " + err.Error() s.eventRecorder.Event(service, "updating loadbalancer failed", message) return err }
func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Service, hosts []string) (*api.LoadBalancerStatus, error) { glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", clusterName, apiService.Namespace, apiService.Name, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, hosts, apiService.Annotations) ports := apiService.Spec.Ports if len(ports) == 0 { return nil, fmt.Errorf("no ports provided to openstack load balancer") } // Check for TCP protocol on each port // TODO: Convert all error messages to use an event recorder for _, port := range ports { if port.Protocol != api.ProtocolTCP { return nil, fmt.Errorf("Only TCP LoadBalancer is supported for openstack load balancers") } } affinity := api.ServiceAffinityNone //apiService.Spec.SessionAffinity var persistence *v2_pools.SessionPersistence switch affinity { case api.ServiceAffinityNone: persistence = nil case api.ServiceAffinityClientIP: persistence = &v2_pools.SessionPersistence{Type: "SOURCE_IP"} default: return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity) } sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) if err != nil { return nil, err } if !service.IsAllowAll(sourceRanges) { return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers") } glog.V(2).Infof("Checking if openstack load balancer already exists: %s", cloudprovider.GetLoadBalancerName(apiService)) _, exists, err := lbaas.GetLoadBalancer(clusterName, apiService) if err != nil { return nil, fmt.Errorf("error checking if openstack load balancer already exists: %v", err) } // TODO: Implement a more efficient update strategy for common changes than delete & create // In particular, if we implement hosts update, we can get rid of UpdateHosts if exists { err := lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) if err != nil { return nil, fmt.Errorf("error deleting existing openstack load balancer: %v", err) } } lbmethod := v2_pools.LBMethod(lbaas.opts.LBMethod) if lbmethod == "" { lbmethod = v2_pools.LBMethodRoundRobin } name := cloudprovider.GetLoadBalancerName(apiService) createOpts := loadbalancers.CreateOpts{ Name: name, Description: fmt.Sprintf("Kubernetes external service %s", name), VipSubnetID: lbaas.opts.SubnetId, } loadBalancerIP := apiService.Spec.LoadBalancerIP if loadBalancerIP != "" { createOpts.VipAddress = loadBalancerIP } loadbalancer, err := loadbalancers.Create(lbaas.network, createOpts).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) for portIndex, port := range ports { listener, err := listeners.Create(lbaas.network, listeners.CreateOpts{ Name: fmt.Sprintf("listener_%s_%d", name, portIndex), Protocol: listeners.Protocol(port.Protocol), ProtocolPort: int(port.Port), LoadbalancerID: loadbalancer.ID, }).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) pool, err := v2_pools.Create(lbaas.network, v2_pools.CreateOpts{ Name: fmt.Sprintf("pool_%s_%d", name, portIndex), Protocol: v2_pools.Protocol(port.Protocol), LBMethod: lbmethod, ListenerID: listener.ID, Persistence: persistence, }).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) for _, host := range hosts { addr, err := getAddressByName(lbaas.compute, host) if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } _, err = v2_pools.CreateAssociateMember(lbaas.network, pool.ID, v2_pools.MemberCreateOpts{ ProtocolPort: int(port.NodePort), Address: addr, SubnetID: lbaas.opts.SubnetId, }).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } if lbaas.opts.CreateMonitor { _, err = v2_monitors.Create(lbaas.network, v2_monitors.CreateOpts{ PoolID: pool.ID, Type: string(port.Protocol), Delay: int(lbaas.opts.MonitorDelay.Duration.Seconds()), Timeout: int(lbaas.opts.MonitorTimeout.Duration.Seconds()), MaxRetries: int(lbaas.opts.MonitorMaxRetries), }).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } } status := &api.LoadBalancerStatus{} status.Ingress = []api.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} if lbaas.opts.FloatingNetworkId != "" { portID, err := getPortIDByIP(lbaas.network, loadbalancer.VipAddress) if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: lbaas.opts.FloatingNetworkId, PortID: portID, } floatIP, err := floatingips.Create(lbaas.network, floatIPOpts).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP}) } return status, nil }
func getPublicIPName(clusterName string, service *v1.Service) string { return fmt.Sprintf("%s-%s", clusterName, cloudprovider.GetLoadBalancerName(service)) }
func getFrontendIPConfigName(service *v1.Service) string { return cloudprovider.GetLoadBalancerName(service) }
// This returns a prefix for loadbalancer/security rules. func getRulePrefix(service *v1.Service) string { return cloudprovider.GetLoadBalancerName(service) }
func (lb *LoadBalancer) EnsureLoadBalancerDeleted(service *api.Service) error { loadBalancerName := cloudprovider.GetLoadBalancerName(service) glog.V(4).Infof("EnsureLoadBalancerDeleted(%v)", loadBalancerName) vip, err := getVipByName(lb.network, loadBalancerName) if err != nil && err != ErrNotFound { return err } if lb.opts.FloatingNetworkId != "" && vip != nil { floatingIP, err := getFloatingIPByPortID(lb.network, vip.PortID) if err != nil && !isNotFound(err) { return err } if floatingIP != nil { err = floatingips.Delete(lb.network, floatingIP.ID).ExtractErr() if err != nil && !isNotFound(err) { return err } } } // We have to delete the VIP before the pool can be deleted, // so no point continuing if this fails. if vip != nil { err := vips.Delete(lb.network, vip.ID).ExtractErr() if err != nil && !isNotFound(err) { return err } } var pool *pools.Pool if vip != nil { pool, err = pools.Get(lb.network, vip.PoolID).Extract() if err != nil && !isNotFound(err) { return err } } else { // The VIP is gone, but it is conceivable that a Pool // still exists that we failed to delete on some // previous occasion. Make a best effort attempt to // cleanup any pools with the same name as the VIP. pool, err = getPoolByName(lb.network, service.Name) if err != nil && err != ErrNotFound { return err } } if pool != nil { for _, monId := range pool.MonitorIDs { _, err = pools.DisassociateMonitor(lb.network, pool.ID, monId).Extract() if err != nil { return err } err = monitors.Delete(lb.network, monId).ExtractErr() if err != nil && !isNotFound(err) { return err } } err = pools.Delete(lb.network, pool.ID).ExtractErr() if err != nil && !isNotFound(err) { return err } } return nil }
func (lb *LoadBalancer) EnsureLoadBalancer(apiService *api.Service, hosts []string) (*api.LoadBalancerStatus, error) { glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v)", apiService.Namespace, apiService.Name, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, hosts, apiService.Annotations) ports := apiService.Spec.Ports if len(ports) > 1 { return nil, fmt.Errorf("multiple ports are not yet supported in openstack load balancers") } else if len(ports) == 0 { return nil, fmt.Errorf("no ports provided to openstack load balancer") } // The service controller verified all the protocols match on the ports, just check and use the first one // TODO: Convert all error messages to use an event recorder if ports[0].Protocol != api.ProtocolTCP { return nil, fmt.Errorf("Only TCP LoadBalancer is supported for openstack load balancers") } affinity := apiService.Spec.SessionAffinity var persistence *vips.SessionPersistence switch affinity { case api.ServiceAffinityNone: persistence = nil case api.ServiceAffinityClientIP: persistence = &vips.SessionPersistence{Type: "SOURCE_IP"} default: return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity) } sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) if err != nil { return nil, err } if !service.IsAllowAll(sourceRanges) { return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers") } glog.V(2).Infof("Checking if openstack load balancer already exists: %s", cloudprovider.GetLoadBalancerName(apiService)) _, exists, err := lb.GetLoadBalancer(apiService) if err != nil { return nil, fmt.Errorf("error checking if openstack load balancer already exists: %v", err) } // TODO: Implement a more efficient update strategy for common changes than delete & create // In particular, if we implement hosts update, we can get rid of UpdateHosts if exists { err := lb.EnsureLoadBalancerDeleted(apiService) if err != nil { return nil, fmt.Errorf("error deleting existing openstack load balancer: %v", err) } } lbmethod := lb.opts.LBMethod if lbmethod == "" { lbmethod = pools.LBMethodRoundRobin } name := cloudprovider.GetLoadBalancerName(apiService) pool, err := pools.Create(lb.network, pools.CreateOpts{ Name: name, Protocol: pools.ProtocolTCP, SubnetID: lb.opts.SubnetId, LBMethod: lbmethod, }).Extract() if err != nil { return nil, err } for _, host := range hosts { addr, err := getAddressByName(lb.compute, host) if err != nil { return nil, err } _, err = members.Create(lb.network, members.CreateOpts{ PoolID: pool.ID, ProtocolPort: int(ports[0].NodePort), //TODO: need to handle multi-port Address: addr, }).Extract() if err != nil { pools.Delete(lb.network, pool.ID) return nil, err } } var mon *monitors.Monitor if lb.opts.CreateMonitor { mon, err = monitors.Create(lb.network, monitors.CreateOpts{ Type: monitors.TypeTCP, Delay: int(lb.opts.MonitorDelay.Duration.Seconds()), Timeout: int(lb.opts.MonitorTimeout.Duration.Seconds()), MaxRetries: int(lb.opts.MonitorMaxRetries), }).Extract() if err != nil { pools.Delete(lb.network, pool.ID) return nil, err } _, err = pools.AssociateMonitor(lb.network, pool.ID, mon.ID).Extract() if err != nil { monitors.Delete(lb.network, mon.ID) pools.Delete(lb.network, pool.ID) return nil, err } } createOpts := vips.CreateOpts{ Name: name, Description: fmt.Sprintf("Kubernetes external service %s", name), Protocol: "TCP", ProtocolPort: int(ports[0].Port), //TODO: need to handle multi-port PoolID: pool.ID, SubnetID: lb.opts.SubnetId, Persistence: persistence, } loadBalancerIP := apiService.Spec.LoadBalancerIP if loadBalancerIP != "" { createOpts.Address = loadBalancerIP } vip, err := vips.Create(lb.network, createOpts).Extract() if err != nil { if mon != nil { monitors.Delete(lb.network, mon.ID) } pools.Delete(lb.network, pool.ID) return nil, err } status := &api.LoadBalancerStatus{} status.Ingress = []api.LoadBalancerIngress{{IP: vip.Address}} if lb.opts.FloatingNetworkId != "" { floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: lb.opts.FloatingNetworkId, PortID: vip.PortID, } floatIP, err := floatingips.Create(lb.network, floatIPOpts).Extract() if err != nil { return nil, err } status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP}) } return status, nil }
func (lbaas *LbaasV2) UpdateLoadBalancer(clusterName string, service *v1.Service, nodeNames []string) error { loadBalancerName := cloudprovider.GetLoadBalancerName(service) glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v)", clusterName, loadBalancerName, nodeNames) ports := service.Spec.Ports if len(ports) == 0 { return fmt.Errorf("no ports provided to openstack load balancer") } loadbalancer, err := getLoadbalancerByName(lbaas.network, loadBalancerName) if err != nil { return err } if loadbalancer == nil { return fmt.Errorf("Loadbalancer %s does not exist", loadBalancerName) } // Get all listeners for this loadbalancer, by "port key". type portKey struct { Protocol string Port int } lbListeners := make(map[portKey]listeners.Listener) err = listeners.List(lbaas.network, listeners.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { listenersList, err := listeners.ExtractListeners(page) if err != nil { return false, err } for _, l := range listenersList { for _, lb := range l.Loadbalancers { // Double check this Listener belongs to the LB we're updating. Neutron's API filtering // can't be counted on in older releases (i.e Liberty). if loadbalancer.ID == lb.ID { key := portKey{Protocol: l.Protocol, Port: l.ProtocolPort} lbListeners[key] = l break } } } return true, nil }) if err != nil { return err } // Get all pools for this loadbalancer, by listener ID. lbPools := make(map[string]v2pools.Pool) err = v2pools.List(lbaas.network, v2pools.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { poolsList, err := v2pools.ExtractPools(page) if err != nil { return false, err } for _, p := range poolsList { for _, l := range p.Listeners { // Double check this Pool belongs to the LB we're deleting. Neutron's API filtering // can't be counted on in older releases (i.e Liberty). for _, val := range lbListeners { if val.ID == l.ID { lbPools[l.ID] = p break } } } } return true, nil }) if err != nil { return err } // Compose Set of member (addresses) that _should_ exist addrs := map[string]empty{} for _, nodeName := range nodeNames { addr, err := getAddressByName(lbaas.compute, types.NodeName(nodeName)) if err != nil { return err } addrs[addr] = empty{} } // Check for adding/removing members associated with each port for _, port := range ports { // Get listener associated with this port listener, ok := lbListeners[portKey{ Protocol: string(port.Protocol), Port: int(port.Port), }] if !ok { return fmt.Errorf("Loadbalancer %s does not contain required listener for port %d and protocol %s", loadBalancerName, port.Port, port.Protocol) } // Get pool associated with this listener pool, ok := lbPools[listener.ID] if !ok { return fmt.Errorf("Loadbalancer %s does not contain required pool for listener %s", loadBalancerName, listener.ID) } // Find existing pool members (by address) for this port members := make(map[string]v2pools.Member) err := v2pools.ListAssociateMembers(lbaas.network, pool.ID, v2pools.MemberListOpts{}).EachPage(func(page pagination.Page) (bool, error) { membersList, err := v2pools.ExtractMembers(page) if err != nil { return false, err } for _, member := range membersList { members[member.Address] = member } return true, nil }) if err != nil { return err } // Add any new members for this port for addr := range addrs { if _, ok := members[addr]; ok && members[addr].ProtocolPort == int(port.NodePort) { // Already exists, do not create member continue } _, err := v2pools.CreateAssociateMember(lbaas.network, pool.ID, v2pools.MemberCreateOpts{ Address: addr, ProtocolPort: int(port.NodePort), SubnetID: lbaas.opts.SubnetId, }).Extract() if err != nil { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // Remove any old members for this port for _, member := range members { if _, ok := addrs[member.Address]; ok && member.ProtocolPort == int(port.NodePort) { // Still present, do not delete member continue } err = v2pools.DeleteMember(lbaas.network, pool.ID, member.ID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } } return nil }
func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Service, nodeNames []string) (*v1.LoadBalancerStatus, error) { glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", clusterName, apiService.Namespace, apiService.Name, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, nodeNames, apiService.Annotations) ports := apiService.Spec.Ports if len(ports) == 0 { return nil, fmt.Errorf("no ports provided to openstack load balancer") } // Check for TCP protocol on each port // TODO: Convert all error messages to use an event recorder for _, port := range ports { if port.Protocol != v1.ProtocolTCP { return nil, fmt.Errorf("Only TCP LoadBalancer is supported for openstack load balancers") } } sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) if err != nil { return nil, err } if !service.IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups { return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers without managing security groups") } affinity := v1.ServiceAffinityNone var persistence *v2pools.SessionPersistence switch affinity { case v1.ServiceAffinityNone: persistence = nil case v1.ServiceAffinityClientIP: persistence = &v2pools.SessionPersistence{Type: "SOURCE_IP"} default: return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity) } name := cloudprovider.GetLoadBalancerName(apiService) loadbalancer, err := getLoadbalancerByName(lbaas.network, name) if err != nil { if err != ErrNotFound { return nil, fmt.Errorf("Error getting loadbalancer %s: %v", name, err) } glog.V(2).Infof("Creating loadbalancer %s", name) loadbalancer, err = lbaas.createLoadBalancer(apiService, name) if err != nil { // Unknown error, retry later return nil, fmt.Errorf("Error creating loadbalancer %s: %v", name, err) } } else { glog.V(2).Infof("LoadBalancer %s already exists", name) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) lbmethod := v2pools.LBMethod(lbaas.opts.LBMethod) if lbmethod == "" { lbmethod = v2pools.LBMethodRoundRobin } oldListeners, err := getListenersByLoadBalancerID(lbaas.network, loadbalancer.ID) if err != nil { return nil, fmt.Errorf("Error getting LB %s listeners: %v", name, err) } for portIndex, port := range ports { listener := getListenerForPort(oldListeners, port) if listener == nil { glog.V(4).Infof("Creating listener for port %d", int(port.Port)) listener, err = listeners.Create(lbaas.network, listeners.CreateOpts{ Name: fmt.Sprintf("listener_%s_%d", name, portIndex), Protocol: listeners.Protocol(port.Protocol), ProtocolPort: int(port.Port), LoadbalancerID: loadbalancer.ID, }).Extract() if err != nil { // Unknown error, retry later return nil, fmt.Errorf("Error creating LB listener: %v", err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } glog.V(4).Infof("Listener for %s port %d: %s", string(port.Protocol), int(port.Port), listener.ID) // After all ports have been processed, remaining listeners are removed as obsolete. // Pop valid listeners. oldListeners = popListener(oldListeners, listener.ID) pool, err := getPoolByListenerID(lbaas.network, loadbalancer.ID, listener.ID) if err != nil && err != ErrNotFound { // Unknown error, retry later return nil, fmt.Errorf("Error getting pool for listener %s: %v", listener.ID, err) } if pool == nil { glog.V(4).Infof("Creating pool for listener %s", listener.ID) pool, err = v2pools.Create(lbaas.network, v2pools.CreateOpts{ Name: fmt.Sprintf("pool_%s_%d", name, portIndex), Protocol: v2pools.Protocol(port.Protocol), LBMethod: lbmethod, ListenerID: listener.ID, Persistence: persistence, }).Extract() if err != nil { // Unknown error, retry later return nil, fmt.Errorf("Error creating pool for listener %s: %v", listener.ID, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } glog.V(4).Infof("Pool for listener %s: %s", listener.ID, pool.ID) members, err := getMembersByPoolID(lbaas.network, pool.ID) if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error getting pool members %s: %v", pool.ID, err) } for _, nodeName := range nodeNames { addr, err := getAddressByName(lbaas.compute, types.NodeName(nodeName)) if err != nil { if err == ErrNotFound { // Node failure, do not create member glog.Warningf("Failed to create LB pool member for node %s: %v", nodeName, err) continue } else { return nil, fmt.Errorf("Error getting address for node %s: %v", nodeName, err) } } if !memberExists(members, addr, int(port.NodePort)) { glog.V(4).Infof("Creating member for pool %s", pool.ID) _, err := v2pools.CreateAssociateMember(lbaas.network, pool.ID, v2pools.MemberCreateOpts{ ProtocolPort: int(port.NodePort), Address: addr, SubnetID: lbaas.opts.SubnetId, }).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB pool member for node: %s, %v", nodeName, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } else { // After all members have been processed, remaining members are deleted as obsolete. members = popMember(members, addr, int(port.NodePort)) } glog.V(4).Infof("Ensured pool %s has member for %s at %s", pool.ID, nodeName, addr) } // Delete obsolete members for this pool for _, member := range members { glog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) err := v2pools.DeleteMember(lbaas.network, pool.ID, member.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } monitorID := pool.MonitorID if monitorID == "" && lbaas.opts.CreateMonitor { glog.V(4).Infof("Creating monitor for pool %s", pool.ID) monitor, err := v2monitors.Create(lbaas.network, v2monitors.CreateOpts{ PoolID: pool.ID, Type: string(port.Protocol), Delay: int(lbaas.opts.MonitorDelay.Duration.Seconds()), Timeout: int(lbaas.opts.MonitorTimeout.Duration.Seconds()), MaxRetries: int(lbaas.opts.MonitorMaxRetries), }).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB pool healthmonitor: %v", err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) monitorID = monitor.ID } glog.V(4).Infof("Monitor for pool %s: %s", pool.ID, monitorID) } // All remaining listeners are obsolete, delete for _, listener := range oldListeners { glog.V(4).Infof("Deleting obsolete listener %s:", listener.ID) // get pool for listener pool, err := getPoolByListenerID(lbaas.network, loadbalancer.ID, listener.ID) if err != nil && err != ErrNotFound { return nil, fmt.Errorf("Error getting pool for obsolete listener %s: %v", listener.ID, err) } if pool != nil { // get and delete monitor monitorID := pool.MonitorID if monitorID != "" { glog.V(4).Infof("Deleting obsolete monitor %s for pool %s", monitorID, pool.ID) err = v2monitors.Delete(lbaas.network, monitorID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleting obsolete monitor %s for pool %s: %v", monitorID, pool.ID, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // get and delete pool members members, err := getMembersByPoolID(lbaas.network, pool.ID) if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error getting members for pool %s: %v", pool.ID, err) } if members != nil { for _, member := range members { glog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) err := v2pools.DeleteMember(lbaas.network, pool.ID, member.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } } glog.V(4).Infof("Deleting obsolete pool %s for listener %s", pool.ID, listener.ID) // delete pool err = v2pools.Delete(lbaas.network, pool.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleting obsolete pool %s for listener %s: %v", pool.ID, listener.ID, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete listener err = listeners.Delete(lbaas.network, listener.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleteting obsolete listener: %v", err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) glog.V(2).Infof("Deleted obsolete listener: %s", listener.ID) } status := &v1.LoadBalancerStatus{} status.Ingress = []v1.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} port, err := getPortByIP(lbaas.network, loadbalancer.VipAddress) if err != nil { return nil, fmt.Errorf("Error getting port for LB vip %s: %v", loadbalancer.VipAddress, err) } floatIP, err := getFloatingIPByPortID(lbaas.network, port.ID) if err != nil && err != ErrNotFound { return nil, fmt.Errorf("Error getting floating ip for port %s: %v", port.ID, err) } if floatIP == nil && lbaas.opts.FloatingNetworkId != "" { glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, port.ID) floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: lbaas.opts.FloatingNetworkId, PortID: port.ID, } floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB floatingip %+v: %v", floatIPOpts, err) } } if floatIP != nil { status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{IP: floatIP.FloatingIP}) } if lbaas.opts.ManageSecurityGroups { lbSecGroupCreateOpts := groups.CreateOpts{ Name: getSecurityGroupName(clusterName, apiService), Description: fmt.Sprintf("Securty Group for %v Service LoadBalancer", apiService.Name), } lbSecGroup, err := groups.Create(lbaas.network, lbSecGroupCreateOpts).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } for _, port := range ports { for _, sourceRange := range sourceRanges.StringSlice() { ethertype := "IPv4" network, _, err := net.ParseCIDR(sourceRange) if err != nil { // cleanup what was created so far glog.Errorf("Error parsing source range %s as a CIDR", sourceRange) _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } if network.To4() == nil { ethertype = "IPv6" } lbSecGroupRuleCreateOpts := rules.CreateOpts{ Direction: "ingress", PortRangeMax: int(port.Port), PortRangeMin: int(port.Port), Protocol: strings.ToLower(string(port.Protocol)), RemoteIPPrefix: sourceRange, SecGroupID: lbSecGroup.ID, EtherType: ethertype, } _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } } err := createNodeSecurityGroup(lbaas.network, lbaas.opts.NodeSecurityGroupID, int(port.NodePort), string(port.Protocol), lbSecGroup.ID) if err != nil { glog.Errorf("Error occured creating security group for loadbalancer %s:", loadbalancer.ID) _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } } lbSecGroupRuleCreateOpts := rules.CreateOpts{ Direction: "ingress", PortRangeMax: 4, // ICMP: Code - Values for ICMP "Destination Unreachable: Fragmentation Needed and Don't Fragment was Set" PortRangeMin: 3, // ICMP: Type Protocol: "icmp", RemoteIPPrefix: "0.0.0.0/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all SecGroupID: lbSecGroup.ID, EtherType: "IPv4", } _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } lbSecGroupRuleCreateOpts = rules.CreateOpts{ Direction: "ingress", PortRangeMax: 0, // ICMP: Code - Values for ICMP "Packet Too Big" PortRangeMin: 2, // ICMP: Type Protocol: "icmp", RemoteIPPrefix: "::/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all SecGroupID: lbSecGroup.ID, EtherType: "IPv6", } _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } // Get the port ID port, err := getPortByIP(lbaas.network, loadbalancer.VipAddress) if err != nil { // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, err } update_opts := neutronports.UpdateOpts{SecurityGroups: []string{lbSecGroup.ID}} res := neutronports.Update(lbaas.network, port.ID, update_opts) if res.Err != nil { glog.Errorf("Error occured updating port: %s", port.ID) // cleanup what was created so far _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) return nil, res.Err } } return status, nil }
func (lbaas *LbaasV2) UpdateLoadBalancer(service *api.Service, hosts []string) error { loadBalancerName := cloudprovider.GetLoadBalancerName(service) glog.V(4).Infof("UpdateLoadBalancer(%v, %v)", loadBalancerName, hosts) ports := service.Spec.Ports if len(ports) > 1 { return fmt.Errorf("multiple ports are not yet supported in openstack load balancers") } else if len(ports) == 0 { return fmt.Errorf("no ports provided to openstack load balancer") } loadbalancer, err := getLoadbalancerByName(lbaas.network, loadBalancerName) if err != nil { return err } if loadbalancer == nil { return fmt.Errorf("Loadbalancer %s does not exist", loadBalancerName) } // Set of member (addresses) that _should_ exist addrs := map[string]bool{} for _, host := range hosts { addr, err := getAddressByName(lbaas.compute, host) if err != nil { return err } addrs[addr] = true } // Iterate over members in each pool that _do_ exist var poolID string err = v2_pools.List(lbaas.network, v2_pools.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { poolsList, err := v2_pools.ExtractPools(page) if err != nil { return false, err } for _, pool := range poolsList { poolID = pool.ID err := v2_pools.ListAssociateMembers(lbaas.network, poolID, v2_pools.MemberListOpts{}).EachPage(func(page pagination.Page) (bool, error) { membersList, err := v2_pools.ExtractMembers(page) if err != nil { return false, err } for _, member := range membersList { if _, found := addrs[member.Address]; found { // Member already exists, remove from update list delete(addrs, member.Address) } else { // Member needs to be deleted err = v2_pools.DeleteMember(lbaas.network, poolID, member.ID).ExtractErr() if err != nil { return false, err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } } return true, nil }) if err != nil { return false, err } } return true, nil }) if err != nil { return err } // Anything left in addrs is a new member that needs to be added to a pool for addr := range addrs { _, err := v2_pools.CreateAssociateMember(lbaas.network, poolID, v2_pools.MemberCreateOpts{ Address: addr, ProtocolPort: int(ports[0].NodePort), SubnetID: lbaas.opts.SubnetId, }).Extract() if err != nil { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } return nil }
func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(clusterName string, service *v1.Service) error { loadBalancerName := cloudprovider.GetLoadBalancerName(service) glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v)", clusterName, loadBalancerName) loadbalancer, err := getLoadbalancerByName(lbaas.network, loadBalancerName) if err != nil && err != ErrNotFound { return err } if loadbalancer == nil { return nil } if lbaas.opts.FloatingNetworkId != "" && loadbalancer != nil { portID, err := getPortIDByIP(lbaas.network, loadbalancer.VipAddress) if err != nil { return err } floatingIP, err := getFloatingIPByPortID(lbaas.network, portID) if err != nil && err != ErrNotFound { return err } if floatingIP != nil { err = floatingips.Delete(lbaas.network, floatingIP.ID).ExtractErr() if err != nil && !isNotFound(err) { return err } } } // get all listeners associated with this loadbalancer var listenerIDs []string err = listeners.List(lbaas.network, listeners.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { listenerList, err := listeners.ExtractListeners(page) if err != nil { return false, err } for _, listener := range listenerList { listenerIDs = append(listenerIDs, listener.ID) } return true, nil }) if err != nil { return err } // get all pools (and health monitors) associated with this loadbalancer var poolIDs []string var monitorIDs []string err = v2pools.List(lbaas.network, v2pools.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { poolsList, err := v2pools.ExtractPools(page) if err != nil { return false, err } for _, pool := range poolsList { poolIDs = append(poolIDs, pool.ID) monitorIDs = append(monitorIDs, pool.MonitorID) } return true, nil }) if err != nil { return err } // get all members associated with each poolIDs var memberIDs []string for _, poolID := range poolIDs { err := v2pools.ListAssociateMembers(lbaas.network, poolID, v2pools.MemberListOpts{}).EachPage(func(page pagination.Page) (bool, error) { membersList, err := v2pools.ExtractMembers(page) if err != nil { return false, err } for _, member := range membersList { memberIDs = append(memberIDs, member.ID) } return true, nil }) if err != nil { return err } } // delete all monitors for _, monitorID := range monitorIDs { err := v2monitors.Delete(lbaas.network, monitorID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete all members and pools for _, poolID := range poolIDs { // delete all members for this pool for _, memberID := range memberIDs { err := v2pools.DeleteMember(lbaas.network, poolID, memberID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete pool err := v2pools.Delete(lbaas.network, poolID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete all listeners for _, listenerID := range listenerIDs { err := listeners.Delete(lbaas.network, listenerID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete loadbalancer err = loadbalancers.Delete(lbaas.network, loadbalancer.ID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerDeleted(lbaas.network, loadbalancer.ID) // Delete the Security Group if lbaas.opts.ManageSecurityGroups { // Generate Name lbSecGroupName := getSecurityGroupName(clusterName, service) lbSecGroupID, err := groups.IDFromName(lbaas.network, lbSecGroupName) if err != nil { glog.V(1).Infof("Error occurred finding security group: %s: %v", lbSecGroupName, err) return nil } lbSecGroup := groups.Delete(lbaas.network, lbSecGroupID) if lbSecGroup.Err != nil && !isNotFound(lbSecGroup.Err) { return lbSecGroup.Err } // Delete the rules in the Node Security Group opts := rules.ListOpts{ SecGroupID: lbaas.opts.NodeSecurityGroupID, RemoteGroupID: lbSecGroupID, } secGroupRules, err := getSecurityGroupRules(lbaas.network, opts) if err != nil && !isNotFound(err) { glog.Errorf("Error finding rules for remote group id %s in security group id %s", lbSecGroupID, lbaas.opts.NodeSecurityGroupID) return err } for _, rule := range secGroupRules { res := rules.Delete(lbaas.network, rule.ID) if res.Err != nil && !isNotFound(res.Err) { glog.V(1).Infof("Error occurred deleting security group rule: %s: %v", rule.ID, res.Err) } } } return nil }
func (s *ServiceController) loadBalancerName(service *api.Service) string { return cloudprovider.GetLoadBalancerName(service) }
func (lb *LoadBalancer) UpdateLoadBalancer(service *api.Service, hosts []string) error { loadBalancerName := cloudprovider.GetLoadBalancerName(service) glog.V(4).Infof("UpdateLoadBalancer(%v, %v)", loadBalancerName, hosts) vip, err := getVipByName(lb.network, loadBalancerName) if err != nil { return err } // Set of member (addresses) that _should_ exist addrs := map[string]bool{} for _, host := range hosts { addr, err := getAddressByName(lb.compute, host) if err != nil { return err } addrs[addr] = true } // Iterate over members that _do_ exist pager := members.List(lb.network, members.ListOpts{PoolID: vip.PoolID}) err = pager.EachPage(func(page pagination.Page) (bool, error) { memList, err := members.ExtractMembers(page) if err != nil { return false, err } for _, member := range memList { if _, found := addrs[member.Address]; found { // Member already exists delete(addrs, member.Address) } else { // Member needs to be deleted err = members.Delete(lb.network, member.ID).ExtractErr() if err != nil { return false, err } } } return true, nil }) if err != nil { return err } // Anything left in addrs is a new member that needs to be added for addr := range addrs { _, err := members.Create(lb.network, members.CreateOpts{ PoolID: vip.PoolID, Address: addr, ProtocolPort: vip.ProtocolPort, }).Extract() if err != nil { return err } } return nil }
func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Service, nodeNames []string) (*api.LoadBalancerStatus, error) { glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", clusterName, apiService.Namespace, apiService.Name, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, nodeNames, apiService.Annotations) ports := apiService.Spec.Ports if len(ports) == 0 { return nil, fmt.Errorf("no ports provided to openstack load balancer") } // Check for TCP protocol on each port // TODO: Convert all error messages to use an event recorder for _, port := range ports { if port.Protocol != api.ProtocolTCP { return nil, fmt.Errorf("Only TCP LoadBalancer is supported for openstack load balancers") } } affinity := api.ServiceAffinityNone //apiService.Spec.SessionAffinity var persistence *v2_pools.SessionPersistence switch affinity { case api.ServiceAffinityNone: persistence = nil case api.ServiceAffinityClientIP: persistence = &v2_pools.SessionPersistence{Type: "SOURCE_IP"} default: return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity) } sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) if err != nil { return nil, err } if !service.IsAllowAll(sourceRanges) { return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers") } name := cloudprovider.GetLoadBalancerName(apiService) loadbalancer, err := getLoadbalancerByName(lbaas.network, name) if err != nil { if err != ErrNotFound { return nil, fmt.Errorf("Error getting loadbalancer %s: %v", name, err) } glog.V(2).Infof("Creating loadbalancer %s", name) loadbalancer, err = lbaas.createLoadBalancer(apiService, name) if err != nil { // Unknown error, retry later return nil, fmt.Errorf("Error creating loadbalancer %s: %v", name, err) } } else { glog.V(2).Infof("LoadBalancer %s already exists", name) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) lbmethod := v2_pools.LBMethod(lbaas.opts.LBMethod) if lbmethod == "" { lbmethod = v2_pools.LBMethodRoundRobin } oldListeners, err := getListenersByLoadBalancerID(lbaas.network, loadbalancer.ID) if err != nil { return nil, fmt.Errorf("Error getting LB %s listeners: %v", name, err) } for portIndex, port := range ports { listener := getListenerForPort(oldListeners, port) if listener == nil { glog.V(4).Infof("Creating listener for port %d", int(port.Port)) listener, err = listeners.Create(lbaas.network, listeners.CreateOpts{ Name: fmt.Sprintf("listener_%s_%d", name, portIndex), Protocol: listeners.Protocol(port.Protocol), ProtocolPort: int(port.Port), LoadbalancerID: loadbalancer.ID, }).Extract() if err != nil { // Unknown error, retry later return nil, fmt.Errorf("Error creating LB listener: %v", err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } glog.V(4).Infof("Listener for %s port %d: %s", string(port.Protocol), int(port.Port), listener.ID) // After all ports have been processed, remaining listeners are removed as obsolete. // Pop valid listeners. oldListeners = popListener(oldListeners, listener.ID) pool, err := getPoolByListenerID(lbaas.network, loadbalancer.ID, listener.ID) if err != nil && err != ErrNotFound { // Unknown error, retry later return nil, fmt.Errorf("Error getting pool for listener %s: %v", listener.ID, err) } if pool == nil { glog.V(4).Infof("Creating pool for listener %s", listener.ID) pool, err = v2_pools.Create(lbaas.network, v2_pools.CreateOpts{ Name: fmt.Sprintf("pool_%s_%d", name, portIndex), Protocol: v2_pools.Protocol(port.Protocol), LBMethod: lbmethod, ListenerID: listener.ID, Persistence: persistence, }).Extract() if err != nil { // Unknown error, retry later return nil, fmt.Errorf("Error creating pool for listener %s: %v", listener.ID, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } glog.V(4).Infof("Pool for listener %s: %s", listener.ID, pool.ID) members, err := getMembersByPoolID(lbaas.network, pool.ID) if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error getting pool members %s: %v", pool.ID, err) } for _, nodeName := range nodeNames { addr, err := getAddressByName(lbaas.compute, types.NodeName(nodeName)) if err != nil { if err == ErrNotFound { // Node failure, do not create member glog.Warningf("Failed to create LB pool member for node %s: %v", nodeName, err) continue } else { return nil, fmt.Errorf("Error getting address for node %s: %v", nodeName, err) } } if !memberExists(members, addr) { glog.V(4).Infof("Creating member for pool %s", pool.ID) _, err := v2_pools.CreateAssociateMember(lbaas.network, pool.ID, v2_pools.MemberCreateOpts{ ProtocolPort: int(port.NodePort), Address: addr, SubnetID: lbaas.opts.SubnetId, }).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB pool member for node: %s, %v", nodeName, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // After all members have been processed, remaining members are deleted as obsolete. members = popMember(members, addr) glog.V(4).Infof("Ensured pool %s has member for %s at %s", pool.ID, nodeName, addr) } // Delete obsolete members for this pool for _, member := range members { glog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) err := v2_pools.DeleteMember(lbaas.network, pool.ID, member.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } monitorID := pool.MonitorID if monitorID == "" && lbaas.opts.CreateMonitor { glog.V(4).Infof("Creating monitor for pool %s", pool.ID) monitor, err := v2_monitors.Create(lbaas.network, v2_monitors.CreateOpts{ PoolID: pool.ID, Type: string(port.Protocol), Delay: int(lbaas.opts.MonitorDelay.Duration.Seconds()), Timeout: int(lbaas.opts.MonitorTimeout.Duration.Seconds()), MaxRetries: int(lbaas.opts.MonitorMaxRetries), }).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB pool healthmonitor: %v", err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) monitorID = monitor.ID } glog.V(4).Infof("Monitor for pool %s: %s", pool.ID, monitorID) } // All remaining listeners are obsolete, delete for _, listener := range oldListeners { glog.V(4).Infof("Deleting obsolete listener %s:", listener.ID) // get pool for listener pool, err := getPoolByListenerID(lbaas.network, loadbalancer.ID, listener.ID) if err != nil && err != ErrNotFound { return nil, fmt.Errorf("Error getting pool for obsolete listener %s: %v", listener.ID, err) } if pool != nil { // get and delete monitor monitorID := pool.MonitorID if monitorID != "" { glog.V(4).Infof("Deleting obsolete monitor %s for pool %s", monitorID, pool.ID) err = v2_monitors.Delete(lbaas.network, monitorID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleting obsolete monitor %s for pool %s: %v", monitorID, pool.ID, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // get and delete pool members members, err := getMembersByPoolID(lbaas.network, pool.ID) if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error getting members for pool %s: %v", pool.ID, err) } if members != nil { for _, member := range members { glog.V(4).Infof("Deleting obsolete member %s for pool %s address %s", member.ID, pool.ID, member.Address) err := v2_pools.DeleteMember(lbaas.network, pool.ID, member.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleting obsolete member %s for pool %s address %s: %v", member.ID, pool.ID, member.Address, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } } glog.V(4).Infof("Deleting obsolete pool %s for listener %s", pool.ID, listener.ID) // delete pool err = v2_pools.Delete(lbaas.network, pool.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleting obsolete pool %s for listener %s: %v", pool.ID, listener.ID, err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete listener err = listeners.Delete(lbaas.network, listener.ID).ExtractErr() if err != nil && !isNotFound(err) { return nil, fmt.Errorf("Error deleteting obsolete listener: %v", err) } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) glog.V(2).Infof("Deleted obsolete listener: %s", listener.ID) } status := &api.LoadBalancerStatus{} status.Ingress = []api.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} portID, err := getPortIDByIP(lbaas.network, loadbalancer.VipAddress) if err != nil { return nil, fmt.Errorf("Error getting port for LB vip %s: %v", loadbalancer.VipAddress, err) } floatIP, err := getFloatingIPByPortID(lbaas.network, portID) if err != nil && err != ErrNotFound { return nil, fmt.Errorf("Error getting floating ip for port %s: %v", portID, err) } if floatIP == nil && lbaas.opts.FloatingNetworkId != "" { glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, portID) floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: lbaas.opts.FloatingNetworkId, PortID: portID, } floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB floatingip %+v: %v", floatIPOpts, err) } } status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP}) return status, nil }
func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(service *api.Service) error { loadBalancerName := cloudprovider.GetLoadBalancerName(service) glog.V(4).Infof("EnsureLoadBalancerDeleted(%v)", loadBalancerName) loadbalancer, err := getLoadbalancerByName(lbaas.network, loadBalancerName) if err != nil && err != ErrNotFound { return err } if loadbalancer == nil { return nil } if lbaas.opts.FloatingNetworkId != "" && loadbalancer != nil { portID, err := getPortIDByIP(lbaas.network, loadbalancer.VipAddress) if err != nil { return err } floatingIP, err := getFloatingIPByPortID(lbaas.network, portID) if err != nil && err != ErrNotFound { return err } if floatingIP != nil { err = floatingips.Delete(lbaas.network, floatingIP.ID).ExtractErr() if err != nil && !isNotFound(err) { return err } } } // get all listeners associated with this loadbalancer var listenerIDs []string err = listeners.List(lbaas.network, listeners.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { listenerList, err := listeners.ExtractListeners(page) if err != nil { return false, err } for _, listener := range listenerList { listenerIDs = append(listenerIDs, listener.ID) } return true, nil }) if err != nil { return err } // get all pools associated with this loadbalancer var poolIDs []string err = v2_pools.List(lbaas.network, v2_pools.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { poolsList, err := v2_pools.ExtractPools(page) if err != nil { return false, err } for _, pool := range poolsList { poolIDs = append(poolIDs, pool.ID) } return true, nil }) if err != nil { return err } // get all members associated with each poolIDs var memberIDs []string for _, poolID := range poolIDs { err := v2_pools.ListAssociateMembers(lbaas.network, poolID, v2_pools.MemberListOpts{}).EachPage(func(page pagination.Page) (bool, error) { membersList, err := v2_pools.ExtractMembers(page) if err != nil { return false, err } for _, member := range membersList { memberIDs = append(memberIDs, member.ID) } return true, nil }) if err != nil { return err } } // get all monitors associated with each poolIDs var monitorIDs []string for _, poolID := range poolIDs { err = v2_monitors.List(lbaas.network, v2_monitors.ListOpts{PoolID: poolID}).EachPage(func(page pagination.Page) (bool, error) { monitorsList, err := v2_monitors.ExtractMonitors(page) if err != nil { return false, err } for _, monitor := range monitorsList { monitorIDs = append(monitorIDs, monitor.ID) } return true, nil }) if err != nil { return err } } // delete all monitors for _, monitorID := range monitorIDs { err := v2_monitors.Delete(lbaas.network, monitorID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete all members and pools for _, poolID := range poolIDs { // delete all members for this pool for _, memberID := range memberIDs { err := v2_pools.DeleteMember(lbaas.network, poolID, memberID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete pool err := v2_pools.Delete(lbaas.network, poolID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete all listeners for _, listenerID := range listenerIDs { err := listeners.Delete(lbaas.network, listenerID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID) } // delete loadbalancer err = loadbalancers.Delete(lbaas.network, loadbalancer.ID).ExtractErr() if err != nil && !isNotFound(err) { return err } waitLoadbalancerDeleted(lbaas.network, loadbalancer.ID) return nil }