func (c *Controller) purgeStaleServiceRefs(fip *types.FloatingIp, refs contrail.ReferenceList, podIdMap map[string]*api.Pod) {
update := false
for _, ref := range refs {
vmi, err := types.VirtualMachineInterfaceByUuid(c.client, ref.Uuid)
if err != nil {
glog.Errorf("%v", err)
continue
}
instanceRefs, err := vmi.GetVirtualMachineRefs()
if err != nil {
glog.Errorf("%v", err)
continue
}
if len(instanceRefs) == 0 {
continue
}
if _, ok := podIdMap[instanceRefs[0].Uuid]; ok {
continue
}
glog.V(3).Infof("Delete reference from pod %s to %s", ref.Uuid, fip.GetFloatingIpAddress())
fip.DeleteVirtualMachineInterface(vmi.GetUuid())
update = true
}
if update {
err := c.client.Update(fip)
if err != nil {
glog.Errorf("%v", err)
}
}
}
func (c *Controller) updateServicePublicIP(service *api.Service) (*types.FloatingIp, error) {
var publicIp *types.FloatingIp = nil
var err error
resourceName := publicIpResourceName(service)
if service.Spec.ExternalIPs != nil {
// Allocate a floating-ip from the public pool.
publicIp, err = c.networkMgr.LocateFloatingIp(
c.networkMgr.GetPublicNetwork(), resourceName, service.Spec.ExternalIPs[0])
} else if service.Spec.Type == api.ServiceTypeLoadBalancer {
publicIp, err = c.networkMgr.LocateFloatingIp(c.networkMgr.GetPublicNetwork(), resourceName, "")
if err == nil {
status := api.LoadBalancerStatus{Ingress: []api.LoadBalancerIngress{
api.LoadBalancerIngress{IP: publicIp.GetFloatingIpAddress()},
}}
if !reflect.DeepEqual(service.Status.LoadBalancer, status) {
service.Status.LoadBalancer = status
_, err := c.kube.Services(service.Namespace).Update(service)
if err != nil {
glog.Infof("Update service %s LB status: %v", service.Name, err)
}
}
}
}
return publicIp, err
}
// Create floating-ip with 2 vmi references, delete it and verify that
// the back_refs are updated as expected.
func TestDeleteRefs(t *testing.T) {
db := NewInMemDatabase()
vmi1 := new(types.VirtualMachineInterface)
vmi1.SetUuid(uuid.New())
vmi1.SetName("port1")
assert.NoError(t, db.Put(vmi1, nil, GetReferenceList(vmi1)))
vmi2 := new(types.VirtualMachineInterface)
vmi2.SetUuid(uuid.New())
vmi2.SetName("port2")
assert.NoError(t, db.Put(vmi2, nil, GetReferenceList(vmi2)))
fip := new(types.FloatingIp)
fip.SetUuid(uuid.New())
fip.SetName("fip")
fip.AddVirtualMachineInterface(vmi1)
fip.AddVirtualMachineInterface(vmi2)
assert.NoError(t, db.Put(fip, nil, GetReferenceList(fip)))
assert.Error(t, db.Delete(vmi1))
result, err := db.GetBackReferences(parseUID(vmi2.GetUuid()), "floating_ip")
assert.NoError(t, err)
assert.Len(t, result, 1)
assert.NoError(t, db.Delete(fip))
result, err = db.GetBackReferences(parseUID(vmi2.GetUuid()), "floating_ip")
assert.NoError(t, err)
assert.Len(t, result, 0)
assert.NoError(t, db.Delete(vmi1))
assert.NoError(t, db.Delete(vmi2))
}
func getFloatingIpToInstanceList(client contrail.ApiClient, fip *types.FloatingIp) ([]string, error) {
vmList := make([]string, 0)
refs, err := fip.GetVirtualMachineInterfaceRefs()
if err != nil {
return vmList, err
}
for _, ref := range refs {
vmi, err := types.VirtualMachineInterfaceByUuid(client, ref.Uuid)
if err != nil {
continue
}
instanceRefs, err := vmi.GetVirtualMachineRefs()
if err != nil || len(instanceRefs) == 0 {
continue
}
vmList = append(vmList, instanceRefs[0].Uuid)
}
return vmList, nil
}
func (m *InstanceManager) AttachFloatingIp(
podName, projectName string, floatingIp *types.FloatingIp) {
fqn := []string{m.config.DefaultDomain, projectName, podName}
obj, err := m.client.FindByName(
"virtual-machine-interface", strings.Join(fqn, ":"))
if err != nil {
glog.Errorf("GET vmi %s: %v", podName, err)
return
}
vmi := obj.(*types.VirtualMachineInterface)
refs, err := floatingIp.GetVirtualMachineInterfaceRefs()
if err != nil {
glog.Errorf("GET floating-ip %s: %v", floatingIp.GetUuid(), err)
return
}
for _, ref := range refs {
if ref.Uuid == vmi.GetUuid() {
return
}
}
floatingIp.AddVirtualMachineInterface(vmi)
err = m.client.Update(floatingIp)
if err != nil {
glog.Errorf("Update floating-ip %s: %v", podName, err)
}
}
func TestConsistencyServiceIp(t *testing.T) {
env := new(TestFramework)
env.SetUp("192.0.2.0/24")
config := env.config
client := env.client
netnsProject := new(types.Project)
netnsProject.SetFQName("domain", []string{"default-domain", "testns"})
client.Create(netnsProject)
service1 := &api.Service{
ObjectMeta: api.ObjectMeta{
Name: "s1",
Namespace: "testns",
Labels: map[string]string{
config.NetworkTag: "services",
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{
"Name": "pod01",
},
ClusterIP: "10.254.42.42",
Type: api.ServiceTypeLoadBalancer,
},
}
service2 := &api.Service{
ObjectMeta: api.ObjectMeta{
Name: "s2",
Namespace: "testns",
Labels: map[string]string{
config.NetworkTag: "services",
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{
"Name": "pod02",
},
ClusterIP: "10.254.42.43",
ExternalIPs: []string{"10.1.4.89"},
},
}
service3 := &api.Service{
ObjectMeta: api.ObjectMeta{
Name: "s3",
Namespace: "testns",
Labels: map[string]string{
config.NetworkTag: "services",
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{
"Name": "pod01",
},
ClusterIP: "10.254.42.44",
},
}
env.Start()
installPods(env, "testns", 3)
env.AddService(service1, "pod01")
env.AddService(service2, "pod02")
env.AddService(service3, "pod01")
env.SyncBarrier()
env.Shutdown()
assert.True(t, env.checker.Check())
pool, err := types.FloatingIpPoolByName(client, "default-domain:testns:service-services:service-services")
assert.NoError(t, err)
vmi, err := types.VirtualMachineInterfaceByName(client, "default-domain:testns:pod01")
assert.NoError(t, err)
vip := new(types.FloatingIp)
fqn := make([]string, len(pool.GetFQName())+1)
copy(fqn, pool.GetFQName())
fqn[len(pool.GetFQName())] = "s4"
vip.SetFQName(vip.GetDefaultParentType(), fqn)
vip.AddVirtualMachineInterface(vmi)
assert.NoError(t, client.Create(vip))
assert.False(t, env.checker.Check())
assert.NoError(t, client.Delete(vip))
assert.True(t, env.checker.Check())
vip, err = types.FloatingIpByName(client, "default-domain:testns:service-services:service-services:s3")
assert.NoError(t, err)
assert.NoError(t, client.Delete(vip))
assert.False(t, env.checker.Check())
}
func (m *NetworkManagerImpl) LocateFloatingIp(network *types.VirtualNetwork, resourceName, targetAddress string) (*types.FloatingIp, error) {
obj, err := m.client.FindByName("floating-ip-pool", makePoolName(network))
if err != nil {
glog.Errorf("Get floating-ip-pool %s: %v", network.GetName(), err)
return nil, err
}
pool := obj.(*types.FloatingIpPool)
fqn := AppendConst(pool.GetFQName(), resourceName)
obj, err = m.client.FindByName("floating-ip", strings.Join(fqn, ":"))
if err == nil {
fip := obj.(*types.FloatingIp)
if targetAddress != "" && fip.GetFloatingIpAddress() != targetAddress {
fip.SetFloatingIpAddress(targetAddress)
err = m.client.Update(fip)
if err != nil {
glog.Errorf("Update floating-ip %s: %v", resourceName, err)
return nil, err
}
}
return fip, nil
}
projectFQN := network.GetFQName()[0 : len(network.GetFQName())-1]
obj, err = m.client.FindByName("project", strings.Join(projectFQN, ":"))
if err != nil {
glog.Errorf("Get project %s: %v", projectFQN[len(projectFQN)-1], err)
return nil, err
}
project := obj.(*types.Project)
fip := new(types.FloatingIp)
fip.SetParent(pool)
fip.SetName(resourceName)
if targetAddress != "" {
fip.SetFloatingIpAddress(targetAddress)
}
fip.AddProject(project)
err = m.client.Create(fip)
if err != nil {
glog.Errorf("Create floating-ip %s: %v", resourceName, err)
return nil, err
}
if targetAddress == "" {
fip, err = types.FloatingIpByUuid(m.client, fip.GetUuid())
}
return fip, err
}
func (c *Controller) updateService(service *api.Service) {
glog.Infof("Update Service %s", service.Name)
serviceName := ServiceName(c.config, service.Labels)
err := c.serviceMgr.Create(service.Namespace, serviceName)
if err != nil {
return
}
pods, err := c.kube.Pods(service.Namespace).List(makeListOptSelector(service.Spec.Selector))
if err != nil {
glog.Errorf("List pods by service %s: %v", service.Name, err)
return
}
var serviceIp *types.FloatingIp = nil
if service.Spec.ClusterIP != "" {
serviceNetwork, err := c.serviceMgr.LocateServiceNetwork(service.Namespace, serviceName)
if err == nil {
serviceIp, err = c.networkMgr.LocateFloatingIp(
serviceNetwork, service.Name, service.Spec.ClusterIP)
}
} else {
serviceNetwork, err := c.serviceMgr.LookupServiceNetwork(service.Namespace, serviceName)
if err == nil {
c.networkMgr.DeleteFloatingIp(serviceNetwork, service.Name)
}
}
publicIp, err := c.updateServicePublicIP(service)
if err == nil && publicIp == nil {
resourceName := publicIpResourceName(service)
c.networkMgr.DeleteFloatingIp(c.networkMgr.GetPublicNetwork(), resourceName)
}
podIdMap := make(map[string]*api.Pod)
for _, pod := range pods.Items {
podIdMap[string(pod.UID)] = &pod
if serviceIp != nil {
// Connect serviceIp to VMI.
c.instanceMgr.AttachFloatingIp(pod.Name, pod.Namespace, serviceIp)
}
if publicIp != nil {
c.instanceMgr.AttachFloatingIp(pod.Name, pod.Namespace, publicIp)
}
}
// Detach the VIPs from pods which are no longer selected.
if serviceIp != nil {
refs, err := serviceIp.GetVirtualMachineInterfaceRefs()
if err == nil {
c.purgeStaleServiceRefs(serviceIp, refs, podIdMap)
}
}
if publicIp != nil {
refs, err := publicIp.GetVirtualMachineInterfaceRefs()
if err == nil {
c.purgeStaleServiceRefs(publicIp, refs, podIdMap)
}
}
}
func TestConsistencyServiceIp(t *testing.T) {
client := createTestClient()
podStore := new(mocks.Store)
serviceStore := new(mocks.Store)
checker := NewConsistencyChecker(client, NewConfig(), podStore, serviceStore, nil)
kube := mocks.NewKubeClient()
controller := NewTestController(kube, client, nil, nil)
config := controller.config
netnsProject := new(types.Project)
netnsProject.SetFQName("domain", []string{"default-domain", "testns"})
client.Create(netnsProject)
installPods(controller, &kube.PodInterface.Mock, &podStore.Mock, "testns", 3)
service1 := &api.Service{
ObjectMeta: api.ObjectMeta{
Name: "s1",
Namespace: "testns",
Labels: map[string]string{
config.NetworkTag: "services",
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{
"app": "pod01",
},
ClusterIP: "10.254.42.42",
Type: api.ServiceTypeLoadBalancer,
},
}
service2 := &api.Service{
ObjectMeta: api.ObjectMeta{
Name: "s2",
Namespace: "testns",
Labels: map[string]string{
config.NetworkTag: "services",
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{
"app": "pod02",
},
ClusterIP: "10.254.42.43",
ExternalIPs: []string{"10.1.4.89"},
},
}
service3 := &api.Service{
ObjectMeta: api.ObjectMeta{
Name: "s3",
Namespace: "testns",
Labels: map[string]string{
config.NetworkTag: "services",
},
},
Spec: api.ServiceSpec{
Selector: map[string]string{
"app": "pod01",
},
ClusterIP: "10.254.42.44",
},
}
kube.ServiceInterface.On("Update", service1).Return(service1, nil)
shutdown := make(chan struct{})
go controller.Run(shutdown)
controller.AddService(service1)
controller.AddService(service2)
controller.AddService(service3)
serviceStore.On("List").Return([]interface{}{service1, service2, service3})
time.Sleep(100 * time.Millisecond)
type shutdownMsg struct {
}
shutdown <- shutdownMsg{}
time.Sleep(100 * time.Millisecond)
assert.True(t, checker.Check())
pool, err := types.FloatingIpPoolByName(client, "default-domain:testns:service-services:service-services")
assert.NoError(t, err)
vmi, err := types.VirtualMachineInterfaceByName(client, "default-domain:testns:pod01")
assert.NoError(t, err)
vip := new(types.FloatingIp)
fqn := make([]string, len(pool.GetFQName())+1)
copy(fqn, pool.GetFQName())
fqn[len(pool.GetFQName())] = "s4"
vip.SetFQName(vip.GetDefaultParentType(), fqn)
vip.AddVirtualMachineInterface(vmi)
assert.NoError(t, client.Create(vip))
assert.False(t, checker.Check())
assert.NoError(t, client.Delete(vip))
assert.True(t, checker.Check())
vip, err = types.FloatingIpByName(client, "default-domain:testns:service-services:service-services:s3")
assert.NoError(t, err)
assert.NoError(t, client.Delete(vip))
assert.False(t, checker.Check())
}