func (lb *LoadBalancer) UpdateTCPLoadBalancer(name, region string, hosts []string) error {
glog.V(4).Infof("UpdateTCPLoadBalancer(%v, %v, %v)", name, region, hosts)
vip, err := getVipByName(lb.network, name)
if err != nil {
return err
}
// Set of member (addresses) that _should_ exist
addrs := map[string]bool{}
for _, host := range hosts {
addr, err := getAddressByName(lb.compute, host)
if err != nil {
return err
}
addrs[addr] = true
}
// Iterate over members that _do_ exist
pager := members.List(lb.network, members.ListOpts{PoolID: vip.PoolID})
err = pager.EachPage(func(page pagination.Page) (bool, error) {
memList, err := members.ExtractMembers(page)
if err != nil {
return false, err
}
for _, member := range memList {
if _, found := addrs[member.Address]; found {
// Member already exists
delete(addrs, member.Address)
} else {
// Member needs to be deleted
err = members.Delete(lb.network, member.ID).ExtractErr()
if err != nil {
return false, err
}
}
}
return true, nil
})
if err != nil {
return err
}
// Anything left in addrs is a new member that needs to be added
for addr := range addrs {
_, err := members.Create(lb.network, members.CreateOpts{
PoolID: vip.PoolID,
Address: addr,
ProtocolPort: vip.ProtocolPort,
}).Extract()
if err != nil {
return err
}
}
return nil
}
func resourceLBPoolV1Create(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
networkingClient, err := config.networkingV2Client(d.Get("region").(string))
if err != nil {
return fmt.Errorf("Error creating OpenStack networking client: %s", err)
}
createOpts := pools.CreateOpts{
Name: d.Get("name").(string),
Protocol: d.Get("protocol").(string),
SubnetID: d.Get("subnet_id").(string),
LBMethod: d.Get("lb_method").(string),
TenantID: d.Get("tenant_id").(string),
}
log.Printf("[DEBUG] Create Options: %#v", createOpts)
p, err := pools.Create(networkingClient, createOpts).Extract()
if err != nil {
return fmt.Errorf("Error creating OpenStack LB pool: %s", err)
}
log.Printf("[INFO] LB Pool ID: %s", p.ID)
log.Printf("[DEBUG] Waiting for OpenStack LB pool (%s) to become available.", p.ID)
stateConf := &resource.StateChangeConf{
Pending: []string{"PENDING_CREATE"},
Target: []string{"ACTIVE"},
Refresh: waitForLBPoolActive(networkingClient, p.ID),
Timeout: 2 * time.Minute,
Delay: 5 * time.Second,
MinTimeout: 3 * time.Second,
}
_, err = stateConf.WaitForState()
if err != nil {
return err
}
d.SetId(p.ID)
if mIDs := resourcePoolMonitorIDsV1(d); mIDs != nil {
for _, mID := range mIDs {
_, err := pools.AssociateMonitor(networkingClient, p.ID, mID).Extract()
if err != nil {
return fmt.Errorf("Error associating monitor (%s) with OpenStack LB pool (%s): %s", mID, p.ID, err)
}
}
}
if memberOpts := resourcePoolMembersV1(d); memberOpts != nil {
for _, memberOpt := range memberOpts {
_, err := members.Create(networkingClient, memberOpt).Extract()
if err != nil {
return fmt.Errorf("Error creating OpenStack LB member: %s", err)
}
}
}
return resourceLBPoolV1Read(d, meta)
}
func resourceLBMemberV1Create(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
networkingClient, err := config.networkingV2Client(d.Get("region").(string))
if err != nil {
return fmt.Errorf("Error creating OpenStack networking client: %s", err)
}
createOpts := members.CreateOpts{
TenantID: d.Get("tenant_id").(string),
PoolID: d.Get("pool_id").(string),
Address: d.Get("address").(string),
ProtocolPort: d.Get("port").(int),
}
log.Printf("[DEBUG] OpenStack LB Member Create Options: %#v", createOpts)
m, err := members.Create(networkingClient, createOpts).Extract()
if err != nil {
return fmt.Errorf("Error creating OpenStack LB member: %s", err)
}
log.Printf("[INFO] LB member ID: %s", m.ID)
log.Printf("[DEBUG] Waiting for OpenStack LB member (%s) to become available.", m.ID)
stateConf := &resource.StateChangeConf{
Pending: []string{"PENDING_CREATE"},
Target: []string{"ACTIVE", "INACTIVE"},
Refresh: waitForLBMemberActive(networkingClient, m.ID),
Timeout: 2 * time.Minute,
Delay: 5 * time.Second,
MinTimeout: 3 * time.Second,
}
_, err = stateConf.WaitForState()
if err != nil {
return err
}
d.SetId(m.ID)
// Due to the way Gophercloud is currently set up, AdminStateUp must be set post-create
updateOpts := members.UpdateOpts{
AdminStateUp: d.Get("admin_state_up").(bool),
}
log.Printf("[DEBUG] OpenStack LB Member Update Options: %#v", createOpts)
m, err = members.Update(networkingClient, m.ID, updateOpts).Extract()
if err != nil {
return fmt.Errorf("Error updating OpenStack LB member: %s", err)
}
return resourceLBMemberV1Read(d, meta)
}
func createMember(t *testing.T, poolID string) string {
m, err := members.Create(base.Client, members.CreateOpts{
Address: "192.168.199.1",
ProtocolPort: 8080,
PoolID: poolID,
}).Extract()
th.AssertNoErr(t, err)
t.Logf("Created member: ID [%s] Status [%s] Weight [%d] Address [%s] Port [%d]",
m.ID, m.Status, m.Weight, m.Address, m.ProtocolPort)
return m.ID
}
func resourceLBPoolV1Create(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
networkingClient, err := config.networkingV2Client(d.Get("region").(string))
if err != nil {
return fmt.Errorf("Error creating OpenStack networking client: %s", err)
}
createOpts := pools.CreateOpts{
Name: d.Get("name").(string),
Protocol: d.Get("protocol").(string),
SubnetID: d.Get("subnet_id").(string),
LBMethod: d.Get("lb_method").(string),
TenantID: d.Get("tenant_id").(string),
}
log.Printf("[DEBUG] Create Options: %#v", createOpts)
p, err := pools.Create(networkingClient, createOpts).Extract()
if err != nil {
return fmt.Errorf("Error creating OpenStack LB pool: %s", err)
}
log.Printf("[INFO] LB Pool ID: %s", p.ID)
d.SetId(p.ID)
if mIDs := resourcePoolMonitorIDsV1(d); mIDs != nil {
for _, mID := range mIDs {
_, err := pools.AssociateMonitor(networkingClient, p.ID, mID).Extract()
if err != nil {
return fmt.Errorf("Error associating monitor (%s) with OpenStack LB pool (%s): %s", mID, p.ID, err)
}
}
}
if memberOpts := resourcePoolMembersV1(d); memberOpts != nil {
for _, memberOpt := range memberOpts {
_, err := members.Create(networkingClient, memberOpt).Extract()
if err != nil {
return fmt.Errorf("Error creating OpenStack LB member: %s", err)
}
}
}
return resourceLBPoolV1Read(d, meta)
}
func resourceLBMemberV1Create(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
networkingClient, err := config.networkingV2Client(d.Get("region").(string))
if err != nil {
return fmt.Errorf("Error creating OpenStack networking client: %s", err)
}
createOpts := members.CreateOpts{
TenantID: d.Get("tenant_id").(string),
PoolID: d.Get("pool_id").(string),
Address: d.Get("address").(string),
ProtocolPort: d.Get("port").(int),
}
log.Printf("[DEBUG] Create Options: %#v", createOpts)
m, err := members.Create(networkingClient, createOpts).Extract()
if err != nil {
return fmt.Errorf("Error creating OpenStack LB member: %s", err)
}
log.Printf("[INFO] LB member ID: %s", m.ID)
log.Printf("[DEBUG] Waiting for OpenStack LB member (%s) to become available.", m.ID)
stateConf := &resource.StateChangeConf{
Target: []string{"ACTIVE"},
Refresh: waitForLBMemberActive(networkingClient, m.ID),
Timeout: 2 * time.Minute,
Delay: 5 * time.Second,
MinTimeout: 3 * time.Second,
}
_, err = stateConf.WaitForState()
if err != nil {
return err
}
d.SetId(m.ID)
return resourceLBMemberV1Read(d, meta)
}
func resourceLBPoolV1Update(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
networkingClient, err := config.networkingV2Client(d.Get("region").(string))
if err != nil {
return fmt.Errorf("Error creating OpenStack networking client: %s", err)
}
var updateOpts pools.UpdateOpts
// If either option changed, update both.
// Gophercloud complains if one is empty.
if d.HasChange("name") || d.HasChange("lb_method") {
updateOpts.Name = d.Get("name").(string)
updateOpts.LBMethod = d.Get("lb_method").(string)
}
log.Printf("[DEBUG] Updating OpenStack LB Pool %s with options: %+v", d.Id(), updateOpts)
_, err = pools.Update(networkingClient, d.Id(), updateOpts).Extract()
if err != nil {
return fmt.Errorf("Error updating OpenStack LB Pool: %s", err)
}
if d.HasChange("monitor_ids") {
oldMIDsRaw, newMIDsRaw := d.GetChange("security_groups")
oldMIDsSet, newMIDsSet := oldMIDsRaw.(*schema.Set), newMIDsRaw.(*schema.Set)
monitorsToAdd := newMIDsSet.Difference(oldMIDsSet)
monitorsToRemove := oldMIDsSet.Difference(newMIDsSet)
log.Printf("[DEBUG] Monitors to add: %v", monitorsToAdd)
log.Printf("[DEBUG] Monitors to remove: %v", monitorsToRemove)
for _, m := range monitorsToAdd.List() {
_, err := pools.AssociateMonitor(networkingClient, d.Id(), m.(string)).Extract()
if err != nil {
return fmt.Errorf("Error associating monitor (%s) with OpenStack server (%s): %s", m.(string), d.Id(), err)
}
log.Printf("[DEBUG] Associated monitor (%s) with pool (%s)", m.(string), d.Id())
}
for _, m := range monitorsToRemove.List() {
_, err := pools.DisassociateMonitor(networkingClient, d.Id(), m.(string)).Extract()
if err != nil {
return fmt.Errorf("Error disassociating monitor (%s) from OpenStack server (%s): %s", m.(string), d.Id(), err)
}
log.Printf("[DEBUG] Disassociated monitor (%s) from pool (%s)", m.(string), d.Id())
}
}
if d.HasChange("member") {
oldMembersRaw, newMembersRaw := d.GetChange("member")
oldMembersSet, newMembersSet := oldMembersRaw.(*schema.Set), newMembersRaw.(*schema.Set)
membersToAdd := newMembersSet.Difference(oldMembersSet)
membersToRemove := oldMembersSet.Difference(newMembersSet)
log.Printf("[DEBUG] Members to add: %v", membersToAdd)
log.Printf("[DEBUG] Members to remove: %v", membersToRemove)
for _, m := range membersToRemove.List() {
oldMember := resourcePoolMemberV1(d, m)
listOpts := members.ListOpts{
PoolID: d.Id(),
Address: oldMember.Address,
ProtocolPort: oldMember.ProtocolPort,
}
err = members.List(networkingClient, listOpts).EachPage(func(page pagination.Page) (bool, error) {
extractedMembers, err := members.ExtractMembers(page)
if err != nil {
return false, err
}
for _, member := range extractedMembers {
err := members.Delete(networkingClient, member.ID).ExtractErr()
if err != nil {
return false, fmt.Errorf("Error deleting member (%s) from OpenStack LB pool (%s): %s", member.ID, d.Id(), err)
}
log.Printf("[DEBUG] Deleted member (%s) from pool (%s)", member.ID, d.Id())
}
return true, nil
})
}
for _, m := range membersToAdd.List() {
createOpts := resourcePoolMemberV1(d, m)
newMember, err := members.Create(networkingClient, createOpts).Extract()
if err != nil {
return fmt.Errorf("Error creating LB member: %s", err)
}
log.Printf("[DEBUG] Created member (%s) in OpenStack LB pool (%s)", newMember.ID, d.Id())
}
}
return resourceLBPoolV1Read(d, meta)
}
func (lb *LoadBalancer) CreateTCPLoadBalancer(name, region string, externalIP net.IP, ports []*api.ServicePort, hosts []string, affinity api.ServiceAffinity) (*api.LoadBalancerStatus, error) {
glog.V(4).Infof("CreateTCPLoadBalancer(%v, %v, %v, %v, %v, %v)", name, region, externalIP, ports, hosts, affinity)
if len(ports) > 1 {
return nil, fmt.Errorf("multiple ports are not yet supported in openstack load balancers")
}
var persistence *vips.SessionPersistence
switch affinity {
case api.ServiceAffinityNone:
persistence = nil
case api.ServiceAffinityClientIP:
persistence = &vips.SessionPersistence{Type: "SOURCE_IP"}
default:
return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity)
}
lbmethod := lb.opts.LBMethod
if lbmethod == "" {
lbmethod = pools.LBMethodRoundRobin
}
pool, err := pools.Create(lb.network, pools.CreateOpts{
Name: name,
Protocol: pools.ProtocolTCP,
SubnetID: lb.opts.SubnetId,
LBMethod: lbmethod,
}).Extract()
if err != nil {
return nil, err
}
for _, host := range hosts {
addr, err := getAddressByName(lb.compute, host)
if err != nil {
return nil, err
}
_, err = members.Create(lb.network, members.CreateOpts{
PoolID: pool.ID,
ProtocolPort: ports[0].Port, //TODO: need to handle multi-port
Address: addr,
}).Extract()
if err != nil {
pools.Delete(lb.network, pool.ID)
return nil, err
}
}
var mon *monitors.Monitor
if lb.opts.CreateMonitor {
mon, err = monitors.Create(lb.network, monitors.CreateOpts{
Type: monitors.TypeTCP,
Delay: int(lb.opts.MonitorDelay.Duration.Seconds()),
Timeout: int(lb.opts.MonitorTimeout.Duration.Seconds()),
MaxRetries: int(lb.opts.MonitorMaxRetries),
}).Extract()
if err != nil {
pools.Delete(lb.network, pool.ID)
return nil, err
}
_, err = pools.AssociateMonitor(lb.network, pool.ID, mon.ID).Extract()
if err != nil {
monitors.Delete(lb.network, mon.ID)
pools.Delete(lb.network, pool.ID)
return nil, err
}
}
createOpts := vips.CreateOpts{
Name: name,
Description: fmt.Sprintf("Kubernetes external service %s", name),
Protocol: "TCP",
ProtocolPort: ports[0].Port, //TODO: need to handle multi-port
PoolID: pool.ID,
Persistence: persistence,
}
if !externalIP.IsUnspecified() {
createOpts.Address = externalIP.String()
}
vip, err := vips.Create(lb.network, createOpts).Extract()
if err != nil {
if mon != nil {
monitors.Delete(lb.network, mon.ID)
}
pools.Delete(lb.network, pool.ID)
return nil, err
}
status := &api.LoadBalancerStatus{}
status.Ingress = []api.LoadBalancerIngress{{IP: vip.Address}}
return status, nil
}
func (lb *LoadBalancer) EnsureLoadBalancer(name, region string, loadBalancerIP net.IP, ports []*api.ServicePort, hosts []string, serviceName types.NamespacedName, affinity api.ServiceAffinity, annotations map[string]string) (*api.LoadBalancerStatus, error) {
glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", name, region, loadBalancerIP, ports, hosts, serviceName, annotations)
if len(ports) > 1 {
return nil, fmt.Errorf("multiple ports are not yet supported in openstack load balancers")
} else if len(ports) == 0 {
return nil, fmt.Errorf("no ports provided to openstack load balancer")
}
// The service controller verified all the protocols match on the ports, just check and use the first one
// TODO: Convert all error messages to use an event recorder
if ports[0].Protocol != api.ProtocolTCP {
return nil, fmt.Errorf("Only TCP LoadBalancer is supported for openstack load balancers")
}
var persistence *vips.SessionPersistence
switch affinity {
case api.ServiceAffinityNone:
persistence = nil
case api.ServiceAffinityClientIP:
persistence = &vips.SessionPersistence{Type: "SOURCE_IP"}
default:
return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity)
}
sourceRanges, err := service.GetLoadBalancerSourceRanges(annotations)
if err != nil {
return nil, err
}
if !service.IsAllowAll(sourceRanges) {
return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers")
}
glog.V(2).Infof("Checking if openstack load balancer already exists: %s", name)
_, exists, err := lb.GetLoadBalancer(name, region)
if err != nil {
return nil, fmt.Errorf("error checking if openstack load balancer already exists: %v", err)
}
// TODO: Implement a more efficient update strategy for common changes than delete & create
// In particular, if we implement hosts update, we can get rid of UpdateHosts
if exists {
err := lb.EnsureLoadBalancerDeleted(name, region)
if err != nil {
return nil, fmt.Errorf("error deleting existing openstack load balancer: %v", err)
}
}
lbmethod := lb.opts.LBMethod
if lbmethod == "" {
lbmethod = pools.LBMethodRoundRobin
}
pool, err := pools.Create(lb.network, pools.CreateOpts{
Name: name,
Protocol: pools.ProtocolTCP,
SubnetID: lb.opts.SubnetId,
LBMethod: lbmethod,
}).Extract()
if err != nil {
return nil, err
}
for _, host := range hosts {
addr, err := getAddressByName(lb.compute, host)
if err != nil {
return nil, err
}
_, err = members.Create(lb.network, members.CreateOpts{
PoolID: pool.ID,
ProtocolPort: ports[0].NodePort, //TODO: need to handle multi-port
Address: addr,
}).Extract()
if err != nil {
pools.Delete(lb.network, pool.ID)
return nil, err
}
}
var mon *monitors.Monitor
if lb.opts.CreateMonitor {
mon, err = monitors.Create(lb.network, monitors.CreateOpts{
Type: monitors.TypeTCP,
Delay: int(lb.opts.MonitorDelay.Duration.Seconds()),
Timeout: int(lb.opts.MonitorTimeout.Duration.Seconds()),
MaxRetries: int(lb.opts.MonitorMaxRetries),
}).Extract()
if err != nil {
pools.Delete(lb.network, pool.ID)
return nil, err
}
_, err = pools.AssociateMonitor(lb.network, pool.ID, mon.ID).Extract()
if err != nil {
monitors.Delete(lb.network, mon.ID)
pools.Delete(lb.network, pool.ID)
return nil, err
}
}
createOpts := vips.CreateOpts{
Name: name,
Description: fmt.Sprintf("Kubernetes external service %s", name),
Protocol: "TCP",
ProtocolPort: ports[0].Port, //TODO: need to handle multi-port
PoolID: pool.ID,
SubnetID: lb.opts.SubnetId,
Persistence: persistence,
}
if loadBalancerIP != nil {
createOpts.Address = loadBalancerIP.String()
}
vip, err := vips.Create(lb.network, createOpts).Extract()
if err != nil {
if mon != nil {
monitors.Delete(lb.network, mon.ID)
}
pools.Delete(lb.network, pool.ID)
return nil, err
}
status := &api.LoadBalancerStatus{}
status.Ingress = []api.LoadBalancerIngress{{IP: vip.Address}}
if lb.opts.FloatingNetworkId != "" {
floatIPOpts := floatingips.CreateOpts{
FloatingNetworkID: lb.opts.FloatingNetworkId,
PortID: vip.PortID,
}
floatIP, err := floatingips.Create(lb.network, floatIPOpts).Extract()
if err != nil {
return nil, err
}
status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP})
}
return status, nil
}
func (lb *LbaasV1) UpdateLoadBalancer(clusterName string, service *v1.Service, nodeNames []string) error {
loadBalancerName := cloudprovider.GetLoadBalancerName(service)
glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v)", clusterName, loadBalancerName, nodeNames)
vip, err := getVipByName(lb.network, loadBalancerName)
if err != nil {
return err
}
// Set of member (addresses) that _should_ exist
addrs := map[string]bool{}
for _, nodeName := range nodeNames {
addr, err := getAddressByName(lb.compute, types.NodeName(nodeName))
if err != nil {
return err
}
addrs[addr] = true
}
// Iterate over members that _do_ exist
pager := members.List(lb.network, members.ListOpts{PoolID: vip.PoolID})
err = pager.EachPage(func(page pagination.Page) (bool, error) {
memList, err := members.ExtractMembers(page)
if err != nil {
return false, err
}
for _, member := range memList {
if _, found := addrs[member.Address]; found {
// Member already exists
delete(addrs, member.Address)
} else {
// Member needs to be deleted
err = members.Delete(lb.network, member.ID).ExtractErr()
if err != nil {
return false, err
}
}
}
return true, nil
})
if err != nil {
return err
}
// Anything left in addrs is a new member that needs to be added
for addr := range addrs {
_, err := members.Create(lb.network, members.CreateOpts{
PoolID: vip.PoolID,
Address: addr,
ProtocolPort: vip.ProtocolPort,
}).Extract()
if err != nil {
return err
}
}
return nil
}
func (lb *LoadBalancer) CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string, affinity api.AffinityType) (net.IP, error) {
glog.V(2).Infof("CreateTCPLoadBalancer(%v, %v, %v, %v, %v)", name, region, externalIP, port, hosts)
if affinity != api.AffinityTypeNone {
return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity)
}
pool, err := pools.Create(lb.network, pools.CreateOpts{
Name: name,
Protocol: pools.ProtocolTCP,
SubnetID: lb.opts.SubnetId,
}).Extract()
if err != nil {
return nil, err
}
for _, host := range hosts {
addr, err := getAddressByName(lb.compute, host)
if err != nil {
return nil, err
}
_, err = members.Create(lb.network, members.CreateOpts{
PoolID: pool.ID,
ProtocolPort: port,
Address: addr,
}).Extract()
if err != nil {
pools.Delete(lb.network, pool.ID)
return nil, err
}
}
var mon *monitors.Monitor
if lb.opts.CreateMonitor {
mon, err = monitors.Create(lb.network, monitors.CreateOpts{
Type: monitors.TypeTCP,
Delay: int(lb.opts.MonitorDelay.Duration.Seconds()),
Timeout: int(lb.opts.MonitorTimeout.Duration.Seconds()),
MaxRetries: int(lb.opts.MonitorMaxRetries),
}).Extract()
if err != nil {
pools.Delete(lb.network, pool.ID)
return nil, err
}
_, err = pools.AssociateMonitor(lb.network, pool.ID, mon.ID).Extract()
if err != nil {
monitors.Delete(lb.network, mon.ID)
pools.Delete(lb.network, pool.ID)
return nil, err
}
}
vip, err := vips.Create(lb.network, vips.CreateOpts{
Name: name,
Description: fmt.Sprintf("Kubernetes external service %s", name),
Address: externalIP.String(),
Protocol: "TCP",
ProtocolPort: port,
PoolID: pool.ID,
}).Extract()
if err != nil {
if mon != nil {
monitors.Delete(lb.network, mon.ID)
}
pools.Delete(lb.network, pool.ID)
return nil, err
}
return net.ParseIP(vip.Address), nil
}