// Wait till the passFunc confirms that the object it expects to see is in the store.
// Used to observe reflected events.
func waitForReflection(t *testing.T, s cache.Store, key string, passFunc func(n interface{}) bool) error {
nodes := []*api.Node{}
err := wait.Poll(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
if n, _, err := s.GetByKey(key); err == nil && passFunc(n) {
return true, nil
} else {
if err != nil {
t.Errorf("Unexpected error: %v", err)
} else {
if n == nil {
nodes = append(nodes, nil)
} else {
nodes = append(nodes, n.(*api.Node))
}
}
return false, nil
}
})
if err != nil {
t.Logf("Logging consecutive node versions received from store:")
for i, n := range nodes {
t.Logf("%d: %#v", i, n)
}
}
return err
}
func storeVersion(t *testing.T, prefix string, c cache.Store, version string, expectedReturn bool) {
pv := newVolume("pvName", "1Gi", "", "", api.VolumeAvailable, api.PersistentVolumeReclaimDelete)
pv.ResourceVersion = version
ret, err := storeObjectUpdate(c, pv, "volume")
if err != nil {
t.Errorf("%s: expected storeObjectUpdate to succeed, got: %v", prefix, err)
}
if expectedReturn != ret {
t.Errorf("%s: expected storeObjectUpdate to return %v, got: %v", prefix, expectedReturn, ret)
}
// find the stored version
pvObj, found, err := c.GetByKey("pvName")
if err != nil {
t.Errorf("expected volume 'pvName' in the cache, got error instead: %v", err)
}
if !found {
t.Errorf("expected volume 'pvName' in the cache but it was not found")
}
pv, ok := pvObj.(*api.PersistentVolume)
if !ok {
t.Errorf("expected volume in the cache, got different object instead: %+v", pvObj)
}
if ret {
if pv.ResourceVersion != version {
t.Errorf("expected volume with version %s in the cache, got %s instead", version, pv.ResourceVersion)
}
} else {
if pv.ResourceVersion == version {
t.Errorf("expected volume with version other than %s in the cache, got %s instead", version, pv.ResourceVersion)
}
}
}
// Wait till the passFunc confirms that the object it expects to see is in the store.
// Used to observe reflected events.
func waitForReflection(s cache.Store, key string, passFunc func(n interface{}) bool) error {
return wait.Poll(time.Millisecond*10, time.Second*20, func() (bool, error) {
if n, _, err := s.GetByKey(key); err == nil && passFunc(n) {
return true, nil
}
return false, nil
})
}
func lastKnown(reviewRecordStore cache.Store, namespace string) (*reviewRecord, error) {
obj, exists, err := reviewRecordStore.GetByKey(namespace)
if err != nil {
return nil, err
}
if exists {
return obj.(*reviewRecord), nil
}
return nil, nil
}
// deleteNamespaceFromSubjects removes the namespace from each subject
// if no other namespaces are active to that subject, it will also delete the subject from the cache entirely
func deleteNamespaceFromSubjects(subjectRecordStore cache.Store, subjects []string, namespace string) {
for _, subject := range subjects {
obj, exists, _ := subjectRecordStore.GetByKey(subject)
if exists {
subjectRecord := obj.(*subjectRecord)
delete(subjectRecord.namespaces, namespace)
if len(subjectRecord.namespaces) == 0 {
subjectRecordStore.Delete(subjectRecord)
}
}
}
}
// Wait for ingress status to be updated to match the desiredStatus.
func WaitForFedStatusUpdate(t *testing.T, store cache.Store, key string, desiredStatus apiv1.LoadBalancerStatus, timeout time.Duration) error {
retryInterval := 100 * time.Millisecond
err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
obj, found, err := store.GetByKey(key)
if !found || err != nil {
return false, err
}
ingress := obj.(*extensionsv1beta1.Ingress)
return reflect.DeepEqual(ingress.Status.LoadBalancer, desiredStatus), nil
})
return err
}
// cleanupOrphanedPods deletes pods that are bound to nodes that don't
// exist.
func cleanupOrphanedPods(pods []*api.Pod, nodeStore cache.Store, forcefulDeletePodFunc func(*api.Pod) error) {
for _, pod := range pods {
if pod.Spec.NodeName == "" {
continue
}
if _, exists, _ := nodeStore.GetByKey(pod.Spec.NodeName); exists {
continue
}
if err := forcefulDeletePodFunc(pod); err != nil {
utilruntime.HandleError(err)
}
}
}
// addSubjectsToNamespace adds the specified namespace to each subject
func addSubjectsToNamespace(subjectRecordStore cache.Store, subjects []string, namespace string) {
for _, subject := range subjects {
var item *subjectRecord
obj, exists, _ := subjectRecordStore.GetByKey(subject)
if exists {
item = obj.(*subjectRecord)
} else {
item = &subjectRecord{subject: subject, namespaces: sets.NewString()}
subjectRecordStore.Add(item)
}
item.namespaces.Insert(namespace)
}
}
// maybeDeleteTerminatingPod non-gracefully deletes pods that are terminating
// that should not be gracefully terminated.
func (nc *NodeController) maybeDeleteTerminatingPod(obj interface{}, nodeStore cache.Store, forcefulDeletePodFunc func(*api.Pod) error) {
pod, ok := obj.(*api.Pod)
if !ok {
return
}
// consider only terminating pods
if pod.DeletionTimestamp == nil {
return
}
// delete terminating pods that have not yet been scheduled
if len(pod.Spec.NodeName) == 0 {
utilruntime.HandleError(forcefulDeletePodFunc(pod))
return
}
nodeObj, found, err := nodeStore.GetByKey(pod.Spec.NodeName)
if err != nil {
// this can only happen if the Store.KeyFunc has a problem creating
// a key for the pod. If it happens once, it will happen again so
// don't bother requeuing the pod.
utilruntime.HandleError(err)
return
}
// delete terminating pods that have been scheduled on
// nonexistent nodes
if !found {
glog.Warningf("Unable to find Node: %v, deleting all assigned Pods.", pod.Spec.NodeName)
utilruntime.HandleError(forcefulDeletePodFunc(pod))
return
}
// delete terminating pods that have been scheduled on
// nodes that do not support graceful termination
// TODO(mikedanese): this can be removed when we no longer
// guarantee backwards compatibility of master API to kubelets with
// versions less than 1.1.0
node := nodeObj.(*api.Node)
v, err := version.Parse(node.Status.NodeInfo.KubeletVersion)
if err != nil {
glog.V(0).Infof("couldn't parse verions %q of minion: %v", node.Status.NodeInfo.KubeletVersion, err)
utilruntime.HandleError(forcefulDeletePodFunc(pod))
return
}
if gracefulDeletionVersion.GT(v) {
utilruntime.HandleError(forcefulDeletePodFunc(pod))
return
}
}
// Wait for finalizers to appear in federation store.
func WaitForFinalizersInFederationStore(ingressController *IngressController, store cache.Store, key string) error {
retryInterval := 100 * time.Millisecond
timeout := wait.ForeverTestTimeout
err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
obj, found, err := store.GetByKey(key)
if !found || err != nil {
return false, err
}
ingress := obj.(*extensionsv1beta1.Ingress)
if ingressController.hasFinalizerFunc(ingress, apiv1.FinalizerOrphan) &&
ingressController.hasFinalizerFunc(ingress, deletionhelper.FinalizerDeleteFromUnderlyingClusters) {
return true, nil
}
return false, nil
})
return err
}
func getServiceFromEndpoints(serviceStore kubeCache.Store, e *kubeAPI.Endpoints) (*kubeAPI.Service, error) {
var (
err error
key string
obj interface{}
exists bool
ok bool
svc *kubeAPI.Service
)
if key, err = kubeCache.MetaNamespaceKeyFunc(e); err != nil {
return nil, err
}
if obj, exists, err = serviceStore.GetByKey(key); err != nil {
return nil, fmt.Errorf("Error getting service object from services store - %v", err)
}
if !exists {
log.WithFields(log.Fields{"name": e.Name, "namespace": e.Namespace}).Warn("Unable to find service for endpoint")
return nil, nil
}
if svc, ok = obj.(*kubeAPI.Service); !ok {
return nil, fmt.Errorf("got a non service object in services store %v", obj)
}
return svc, nil
}
func getEndpointsForService(endpointsStore kubeCache.Store, s *kubeAPI.Service) (*kubeAPI.Endpoints, error) {
var (
err error
key string
obj interface{}
exists bool
ok bool
e *kubeAPI.Endpoints
)
if key, err = kubeCache.MetaNamespaceKeyFunc(s); err != nil {
return nil, err
}
if obj, exists, err = endpointsStore.GetByKey(key); err != nil {
return nil, fmt.Errorf("Error getting endpoints object from endpoints store - %v", err)
}
if !exists {
log.WithFields(log.Fields{"name": s.Name, "namespace": s.Namespace}).Warn("Unable to find endpoint for service")
return nil, nil
}
if e, ok = obj.(*kubeAPI.Endpoints); !ok {
return nil, fmt.Errorf("got a non endpoints object in endpoints store %v", obj)
}
return e, nil
}