func (factory *RouterControllerFactory) Create(plugin router.Plugin) *controller.RouterController {
routeEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
cache.NewReflector(&routeLW{factory.OSClient}, &routeapi.Route{}, routeEventQueue, 2*time.Minute).Run()
endpointsEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
cache.NewReflector(&endpointsLW{factory.KClient}, &kapi.Endpoints{}, endpointsEventQueue, 2*time.Minute).Run()
return &controller.RouterController{
Plugin: plugin,
NextEndpoints: func() (watch.EventType, *kapi.Endpoints, error) {
eventType, obj, err := endpointsEventQueue.Pop()
if err != nil {
return watch.Error, nil, err
}
return eventType, obj.(*kapi.Endpoints), nil
},
NextRoute: func() (watch.EventType, *routeapi.Route, error) {
eventType, obj, err := routeEventQueue.Pop()
if err != nil {
return watch.Error, nil, err
}
return eventType, obj.(*routeapi.Route), nil
},
}
}
// Create begins listing and watching against the API server for the desired route and endpoint
// resources. It spawns child goroutines that cannot be terminated.
func (factory *RouterControllerFactory) Create(plugin router.Plugin) *controller.RouterController {
routeEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
cache.NewReflector(&routeLW{
client: factory.OSClient,
namespace: factory.Namespace,
field: factory.Fields,
label: factory.Labels,
}, &routeapi.Route{}, routeEventQueue, factory.ResyncInterval).Run()
endpointsEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
cache.NewReflector(&endpointsLW{
client: factory.KClient,
namespace: factory.Namespace,
// we do not scope endpoints by labels or fields because the route labels != endpoints labels
}, &kapi.Endpoints{}, endpointsEventQueue, factory.ResyncInterval).Run()
return &controller.RouterController{
Plugin: plugin,
NextEndpoints: func() (watch.EventType, *kapi.Endpoints, error) {
eventType, obj, err := endpointsEventQueue.Pop()
if err != nil {
return watch.Error, nil, err
}
return eventType, obj.(*kapi.Endpoints), nil
},
NextRoute: func() (watch.EventType, *routeapi.Route, error) {
eventType, obj, err := routeEventQueue.Pop()
if err != nil {
return watch.Error, nil, err
}
return eventType, obj.(*routeapi.Route), nil
},
}
}
// Run event queue for the given resource
func (registry *Registry) RunEventQueue(resourceName ResourceName) *oscache.EventQueue {
var client cache.Getter
var expectedType interface{}
switch resourceName {
case HostSubnets:
expectedType = &osapi.HostSubnet{}
client = registry.oClient
case NetNamespaces:
expectedType = &osapi.NetNamespace{}
client = registry.oClient
case Nodes:
expectedType = &kapi.Node{}
client = registry.kClient
case Namespaces:
expectedType = &kapi.Namespace{}
client = registry.kClient
case Services:
expectedType = &kapi.Service{}
client = registry.kClient
case Pods:
expectedType = &kapi.Pod{}
client = registry.kClient
default:
log.Fatalf("Unknown resource %s during initialization of event queue", resourceName)
}
lw := cache.NewListWatchFromClient(client, strings.ToLower(string(resourceName)), kapi.NamespaceAll, fields.Everything())
eventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
// Repopulate event queue every 30 mins
// Existing items in the event queue will have watch.Modified event type
cache.NewReflector(lw, expectedType, eventQueue, 30*time.Minute).Run()
return eventQueue
}
// Run event queue for the given resource
func (registry *Registry) runEventQueue(resourceName string) (*oscache.EventQueue, *cache.Reflector) {
eventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
lw := &cache.ListWatch{}
var expectedType interface{}
switch strings.ToLower(resourceName) {
case "hostsubnet":
expectedType = &originapi.HostSubnet{}
lw.ListFunc = func(options kapi.ListOptions) (runtime.Object, error) {
return registry.oClient.HostSubnets().List(options)
}
lw.WatchFunc = func(options kapi.ListOptions) (watch.Interface, error) {
return registry.oClient.HostSubnets().Watch(options)
}
case "node":
expectedType = &kapi.Node{}
lw.ListFunc = func(options kapi.ListOptions) (runtime.Object, error) {
return registry.kClient.Nodes().List(options)
}
lw.WatchFunc = func(options kapi.ListOptions) (watch.Interface, error) {
return registry.kClient.Nodes().Watch(options)
}
case "namespace":
expectedType = &kapi.Namespace{}
lw.ListFunc = func(options kapi.ListOptions) (runtime.Object, error) {
return registry.kClient.Namespaces().List(options)
}
lw.WatchFunc = func(options kapi.ListOptions) (watch.Interface, error) {
return registry.kClient.Namespaces().Watch(options)
}
case "netnamespace":
expectedType = &originapi.NetNamespace{}
lw.ListFunc = func(options kapi.ListOptions) (runtime.Object, error) {
return registry.oClient.NetNamespaces().List(options)
}
lw.WatchFunc = func(options kapi.ListOptions) (watch.Interface, error) {
return registry.oClient.NetNamespaces().Watch(options)
}
case "service":
expectedType = &kapi.Service{}
lw.ListFunc = func(options kapi.ListOptions) (runtime.Object, error) {
return registry.kClient.Services(kapi.NamespaceAll).List(options)
}
lw.WatchFunc = func(options kapi.ListOptions) (watch.Interface, error) {
return registry.kClient.Services(kapi.NamespaceAll).Watch(options)
}
case "pod":
expectedType = &kapi.Pod{}
lw.ListFunc = func(options kapi.ListOptions) (runtime.Object, error) {
return registry.kClient.Pods(kapi.NamespaceAll).List(options)
}
lw.WatchFunc = func(options kapi.ListOptions) (watch.Interface, error) {
return registry.kClient.Pods(kapi.NamespaceAll).Watch(options)
}
default:
log.Fatalf("Unknown resource %s during initialization of event queue", resourceName)
}
reflector := cache.NewReflector(lw, expectedType, eventQueue, 4*time.Minute)
reflector.Run()
return eventQueue, reflector
}
func (oi *OsdnRegistryInterface) WatchMinions(receiver chan *osdnapi.MinionEvent, stop chan bool) error {
minionEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
listWatch := &cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return oi.kClient.Nodes().List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return oi.kClient.Nodes().Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
}
cache.NewReflector(listWatch, &kapi.Node{}, minionEventQueue, 4*time.Minute).Run()
for {
eventType, obj, err := minionEventQueue.Pop()
if err != nil {
return err
}
switch eventType {
case watch.Added:
// we should ignore the modified event because status updates cause unnecessary noise
// the only time we would care about modified would be if the minion changes its IP address
// and hence all nodes need to update their vtep entries for the respective subnet
// create minionEvent
node := obj.(*kapi.Node)
receiver <- &osdnapi.MinionEvent{Type: osdnapi.Added, Minion: node.ObjectMeta.Name}
case watch.Deleted:
// TODO: There is a chance that a Delete event will not get triggered.
// Need to use a periodic sync loop that lists and compares.
node := obj.(*kapi.Node)
receiver <- &osdnapi.MinionEvent{Type: osdnapi.Deleted, Minion: node.ObjectMeta.Name}
}
}
return nil
}
func (oi *OsdnRegistryInterface) WatchSubnets(receiver chan *osdnapi.SubnetEvent, stop chan bool) error {
subnetEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
listWatch := &cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return oi.oClient.HostSubnets().List()
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return oi.oClient.HostSubnets().Watch(resourceVersion)
},
}
cache.NewReflector(listWatch, &api.HostSubnet{}, subnetEventQueue, 4*time.Minute).Run()
for {
eventType, obj, err := subnetEventQueue.Pop()
if err != nil {
return err
}
switch eventType {
case watch.Added, watch.Modified:
// create SubnetEvent
hs := obj.(*api.HostSubnet)
receiver <- &osdnapi.SubnetEvent{Type: osdnapi.Added, Minion: hs.Host, Sub: osdnapi.Subnet{Minion: hs.HostIP, Sub: hs.Subnet}}
case watch.Deleted:
// TODO: There is a chance that a Delete event will not get triggered.
// Need to use a periodic sync loop that lists and compares.
hs := obj.(*api.HostSubnet)
receiver <- &osdnapi.SubnetEvent{Type: osdnapi.Deleted, Minion: hs.Host, Sub: osdnapi.Subnet{Minion: hs.HostIP, Sub: hs.Subnet}}
}
}
return nil
}
// Run event queue for the given resource
func (registry *Registry) runEventQueue(resourceName string) *oscache.EventQueue {
var client cache.Getter
var expectedType interface{}
switch strings.ToLower(resourceName) {
case "hostsubnets":
expectedType = &osapi.HostSubnet{}
client = registry.oClient
case "netnamespaces":
expectedType = &osapi.NetNamespace{}
client = registry.oClient
case "nodes":
expectedType = &kapi.Node{}
client = registry.kClient
case "namespaces":
expectedType = &kapi.Namespace{}
client = registry.kClient
case "services":
expectedType = &kapi.Service{}
client = registry.kClient
case "pods":
expectedType = &kapi.Pod{}
client = registry.kClient
default:
log.Fatalf("Unknown resource %s during initialization of event queue", resourceName)
}
lw := cache.NewListWatchFromClient(client, strings.ToLower(resourceName), kapi.NamespaceAll, fields.Everything())
eventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
cache.NewReflector(lw, expectedType, eventQueue, 0).Run()
return eventQueue
}
// Create begins listing and watching against the API server for the desired route and endpoint
// resources. It spawns child goroutines that cannot be terminated.
func (factory *RouterControllerFactory) Create(plugin router.Plugin) *controller.RouterController {
routeEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
cache.NewReflector(&routeLW{
client: factory.OSClient,
namespace: factory.Namespace,
field: factory.Fields,
label: factory.Labels,
}, &routeapi.Route{}, routeEventQueue, factory.ResyncInterval).Run()
endpointsEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
cache.NewReflector(&endpointsLW{
client: factory.KClient,
namespace: factory.Namespace,
// we do not scope endpoints by labels or fields because the route labels != endpoints labels
}, &kapi.Endpoints{}, endpointsEventQueue, factory.ResyncInterval).Run()
return &controller.RouterController{
Plugin: plugin,
NextEndpoints: func() (watch.EventType, *kapi.Endpoints, error) {
eventType, obj, err := endpointsEventQueue.Pop()
if err != nil {
return watch.Error, nil, err
}
return eventType, obj.(*kapi.Endpoints), nil
},
NextRoute: func() (watch.EventType, *routeapi.Route, error) {
eventType, obj, err := routeEventQueue.Pop()
if err != nil {
return watch.Error, nil, err
}
return eventType, obj.(*routeapi.Route), nil
},
EndpointsListConsumed: func() bool {
return endpointsEventQueue.ListConsumed()
},
RoutesListConsumed: func() bool {
return routeEventQueue.ListConsumed()
},
Namespaces: factory.Namespaces,
// check namespaces a bit more often than we resync events, so that we aren't always waiting
// the maximum interval for new items to come into the list
// TODO: trigger a reflector resync after every namespace sync?
NamespaceSyncInterval: factory.ResyncInterval - 10*time.Second,
NamespaceWaitInterval: 10 * time.Second,
NamespaceRetries: 5,
}
}
// Run event queue for the given resource
func (oi *OsdnRegistryInterface) runEventQueue(resourceName string, args interface{}) (*oscache.EventQueue, *cache.Reflector) {
eventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
lw := &cache.ListWatch{}
var expectedType interface{}
switch strings.ToLower(resourceName) {
case "hostsubnet":
expectedType = &api.HostSubnet{}
lw.ListFunc = func() (runtime.Object, error) {
return oi.oClient.HostSubnets().List()
}
lw.WatchFunc = func(resourceVersion string) (watch.Interface, error) {
return oi.oClient.HostSubnets().Watch(resourceVersion)
}
case "node":
expectedType = &kapi.Node{}
lw.ListFunc = func() (runtime.Object, error) {
return oi.kClient.Nodes().List(labels.Everything(), fields.Everything())
}
lw.WatchFunc = func(resourceVersion string) (watch.Interface, error) {
return oi.kClient.Nodes().Watch(labels.Everything(), fields.Everything(), resourceVersion)
}
case "namespace":
expectedType = &kapi.Namespace{}
lw.ListFunc = func() (runtime.Object, error) {
return oi.kClient.Namespaces().List(labels.Everything(), fields.Everything())
}
lw.WatchFunc = func(resourceVersion string) (watch.Interface, error) {
return oi.kClient.Namespaces().Watch(labels.Everything(), fields.Everything(), resourceVersion)
}
case "netnamespace":
expectedType = &api.NetNamespace{}
lw.ListFunc = func() (runtime.Object, error) {
return oi.oClient.NetNamespaces().List()
}
lw.WatchFunc = func(resourceVersion string) (watch.Interface, error) {
return oi.oClient.NetNamespaces().Watch(resourceVersion)
}
case "service":
expectedType = &kapi.Service{}
namespace := args.(string)
lw.ListFunc = func() (runtime.Object, error) {
return oi.kClient.Services(namespace).List(labels.Everything())
}
lw.WatchFunc = func(resourceVersion string) (watch.Interface, error) {
return oi.kClient.Services(namespace).Watch(labels.Everything(), fields.Everything(), resourceVersion)
}
default:
log.Fatalf("Unknown resource %s during initialization of event queue", resourceName)
}
reflector := cache.NewReflector(lw, expectedType, eventQueue, 4*time.Minute)
reflector.Run()
return eventQueue, reflector
}
func (oi *OsdnRegistryInterface) WatchNodes(receiver chan *osdnapi.NodeEvent, stop chan bool) error {
nodeEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
listWatch := &cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return oi.kClient.Nodes().List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return oi.kClient.Nodes().Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
}
cache.NewReflector(listWatch, &kapi.Node{}, nodeEventQueue, 4*time.Minute).Run()
nodeAddressMap, err := oi.getNodeAddressMap()
if err != nil {
return err
}
for {
eventType, obj, err := nodeEventQueue.Pop()
if err != nil {
return err
}
node := obj.(*kapi.Node)
nodeIP := ""
if len(node.Status.Addresses) > 0 {
nodeIP = node.Status.Addresses[0].Address
} else {
nodeIP, err = osdn.GetNodeIP(node.ObjectMeta.Name)
if err != nil {
return err
}
}
switch eventType {
case watch.Added:
receiver <- &osdnapi.NodeEvent{Type: osdnapi.Added, Node: osdnapi.Node{Name: node.ObjectMeta.Name, IP: nodeIP}}
nodeAddressMap[node.ObjectMeta.UID] = nodeIP
case watch.Modified:
oldNodeIP, ok := nodeAddressMap[node.ObjectMeta.UID]
if ok && oldNodeIP != nodeIP {
// Node Added event will handle update subnet if there is ip mismatch
receiver <- &osdnapi.NodeEvent{Type: osdnapi.Added, Node: osdnapi.Node{Name: node.ObjectMeta.Name, IP: nodeIP}}
nodeAddressMap[node.ObjectMeta.UID] = nodeIP
}
case watch.Deleted:
// TODO: There is a chance that a Delete event will not get triggered.
// Need to use a periodic sync loop that lists and compares.
receiver <- &osdnapi.NodeEvent{Type: osdnapi.Deleted, Node: osdnapi.Node{Name: node.ObjectMeta.Name}}
delete(nodeAddressMap, node.ObjectMeta.UID)
}
}
}
func (oi *OsdnRegistryInterface) watchServicesForNamespace(namespace string, receiver chan *osdnapi.ServiceEvent, stop chan bool) error {
serviceEventQueue := oscache.NewEventQueue(cache.MetaNamespaceKeyFunc)
listWatch := &cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return oi.kClient.Services(namespace).List(labels.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return oi.kClient.Services(namespace).Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
}
cache.NewReflector(listWatch, &kapi.Service{}, serviceEventQueue, 4*time.Minute).Run()
go func() {
select {
case <-stop:
serviceEventQueue.Cancel()
}
}()
for {
eventType, obj, err := serviceEventQueue.Pop()
if err != nil {
if _, ok := err.(oscache.EventQueueStopped); ok {
return nil
}
return err
}
switch eventType {
case watch.Added:
// we should ignore the modified event because status updates cause unnecessary noise
// the only time we would care about modified would be if the service IP changes (does not happen)
kServ := obj.(*kapi.Service)
if kServ.Spec.ClusterIP == "None" {
continue
}
for i := range kServ.Spec.Ports {
oServ := osdnapi.Service{
Name: kServ.ObjectMeta.Name,
Namespace: namespace,
IP: kServ.Spec.ClusterIP,
Protocol: osdnapi.ServiceProtocol(kServ.Spec.Ports[i].Protocol),
Port: uint(kServ.Spec.Ports[i].Port),
}
receiver <- &osdnapi.ServiceEvent{Type: osdnapi.Added, Service: oServ}
}
case watch.Deleted:
// TODO: There is a chance that a Delete event will not get triggered.
// Need to use a periodic sync loop that lists and compares.
kServ := obj.(*kapi.Service)
if kServ.Spec.ClusterIP == "None" {
continue
}
for i := range kServ.Spec.Ports {
oServ := osdnapi.Service{
Name: kServ.ObjectMeta.Name,
Namespace: namespace,
IP: kServ.Spec.ClusterIP,
Protocol: osdnapi.ServiceProtocol(kServ.Spec.Ports[i].Protocol),
Port: uint(kServ.Spec.Ports[i].Port),
}
receiver <- &osdnapi.ServiceEvent{Type: osdnapi.Deleted, Service: oServ}
}
case watch.Error:
// Check if the namespace is dead, if so quit
_, err = oi.kClient.Namespaces().Get(namespace)
if err != nil {
break
}
}
}
}