func waitForAnyPersistentVolumePhase(w watch.Interface, phase api.PersistentVolumePhase) {
for {
event := <-w.ResultChan()
volume, ok := event.Object.(*api.PersistentVolume)
if !ok {
continue
}
if volume.Status.Phase == phase {
glog.V(2).Infof("volume %q is %s", volume.Name, phase)
break
}
}
}
// Until reads items from the watch until each provided condition succeeds, and then returns the last watch
// encountered. The first condition that returns an error terminates the watch (and the event is also returned).
// If no event has been received, the returned event will be nil.
// TODO: move to pkg/watch upstream
func Until(timeout time.Duration, watcher watch.Interface, conditions ...WatchConditionFunc) (*watch.Event, error) {
ch := watcher.ResultChan()
defer watcher.Stop()
var after <-chan time.Time
if timeout > 0 {
after = time.After(timeout)
} else {
ch := make(chan time.Time)
close(ch)
after = ch
}
var lastEvent *watch.Event
for _, condition := range conditions {
for {
select {
case event, ok := <-ch:
if !ok {
return lastEvent, wait.ErrWaitTimeout
}
lastEvent = &event
// TODO: check for watch expired error and retry watch from latest point?
done, err := condition(event)
if err != nil {
return lastEvent, err
}
if done {
return lastEvent, nil
}
case <-after:
return lastEvent, wait.ErrWaitTimeout
}
}
}
return lastEvent, wait.ErrWaitTimeout
}
// watchHandler watches w and keeps *resourceVersion up to date.
func (r *Reflector) watchHandler(w watch.Interface, resourceVersion *string, resyncCh <-chan time.Time, stopCh <-chan struct{}) error {
start := time.Now()
eventCount := 0
// Stopping the watcher should be idempotent and if we return from this function there's no way
// we're coming back in with the same watch interface.
defer w.Stop()
loop:
for {
select {
case <-stopCh:
return errorStopRequested
case <-resyncCh:
return errorResyncRequested
case event, ok := <-w.ResultChan():
if !ok {
break loop
}
if event.Type == watch.Error {
return apierrs.FromObject(event.Object)
}
if e, a := r.expectedType, reflect.TypeOf(event.Object); e != nil && e != a {
utilruntime.HandleError(fmt.Errorf("%s: expected type %v, but watch event object had type %v", r.name, e, a))
continue
}
meta, err := meta.Accessor(event.Object)
if err != nil {
utilruntime.HandleError(fmt.Errorf("%s: unable to understand watch event %#v", r.name, event))
continue
}
newResourceVersion := meta.GetResourceVersion()
switch event.Type {
case watch.Added:
r.store.Add(event.Object)
case watch.Modified:
r.store.Update(event.Object)
case watch.Deleted:
// TODO: Will any consumers need access to the "last known
// state", which is passed in event.Object? If so, may need
// to change this.
r.store.Delete(event.Object)
default:
utilruntime.HandleError(fmt.Errorf("%s: unable to understand watch event %#v", r.name, event))
}
*resourceVersion = newResourceVersion
r.setLastSyncResourceVersion(newResourceVersion)
eventCount++
}
}
watchDuration := time.Now().Sub(start)
if watchDuration < 1*time.Second && eventCount == 0 {
glog.V(4).Infof("%s: Unexpected watch close - watch lasted less than a second and no items received", r.name)
return errors.New("very short watch")
}
glog.V(4).Infof("%s: Watch close - %v total %v items received", r.name, r.expectedType, eventCount)
return nil
}
func waitForWatch(t *testing.T, name string, w watchapi.Interface) *watchapi.Event {
select {
case e := <-w.ResultChan():
return &e
case <-time.After(BuildControllersWatchTimeout):
t.Fatalf("Timed out waiting for watch: %s", name)
return nil
}
}
func waitForPersistentVolumePhase(w watch.Interface, phase api.PersistentVolumePhase) {
for {
event := <-w.ResultChan()
volume := event.Object.(*api.PersistentVolume)
if volume.Status.Phase == phase {
break
}
}
}
func testCheckEventType(t *testing.T, expectEventType watch.EventType, w watch.Interface) {
select {
case res := <-w.ResultChan():
if res.Type != expectEventType {
t.Errorf("event type want=%v, get=%v", expectEventType, res.Type)
}
case <-time.After(wait.ForeverTestTimeout):
t.Errorf("time out after waiting %v on ResultChan", wait.ForeverTestTimeout)
}
}
func observeCreation(w watch.Interface) {
select {
case event, _ := <-w.ResultChan():
if event.Type != watch.Added {
framework.Failf("Failed to observe the creation: %v", event)
}
case <-time.After(30 * time.Second):
framework.Failf("Timeout while waiting for observing the creation")
}
}
func observePodCreation(w watch.Interface) {
select {
case event, _ := <-w.ResultChan():
if event.Type != watch.Added {
framework.Failf("Failed to observe pod creation: %v", event)
}
case <-time.After(framework.PodStartTimeout):
Fail("Timeout while waiting for pod creation")
}
}
func testCheckStop(t *testing.T, i int, w watch.Interface) {
select {
case _, ok := <-w.ResultChan():
if ok {
t.Errorf("#%d: ResultChan should have been closed", i)
}
case <-time.After(wait.ForeverTestTimeout):
t.Errorf("#%d: time out after waiting 1s on ResultChan", i)
}
}
func (i ingester) ingest(in watch.Interface, out chan<- Event, c context.Context) {
for {
select {
case <-c.Done():
logf(i).Debug("Closing ingest channel")
return
case e := <-in.ResultChan():
out <- Event{e}
}
}
}
func waitForAnyPersistentVolumeClaimPhase(w watch.Interface, phase api.PersistentVolumeClaimPhase) {
for {
event := <-w.ResultChan()
claim, ok := event.Object.(*api.PersistentVolumeClaim)
if !ok {
continue
}
if claim.Status.Phase == phase {
glog.V(2).Infof("claim %q is %s", claim.Name, phase)
break
}
}
}
func verifyWatchEvent(t *testing.T, w watch.Interface, eventType watch.EventType, eventObject runtime.Object) {
select {
case event := <-w.ResultChan():
if e, a := eventType, event.Type; e != a {
t.Errorf("Expected: %s, got: %s", eventType, event.Type)
}
if e, a := eventObject, event.Object; !api.Semantic.DeepDerivative(e, a) {
t.Errorf("Expected (%s): %#v, got: %#v", eventType, e, a)
}
case <-time.After(wait.ForeverTestTimeout):
t.Errorf("Timed out waiting for an event")
}
}
func testCheckResult(t *testing.T, i int, expectEventType watch.EventType, w watch.Interface, expectObj *api.Pod) {
select {
case res := <-w.ResultChan():
if res.Type != expectEventType {
t.Errorf("#%d: event type want=%v, get=%v", i, expectEventType, res.Type)
return
}
if !reflect.DeepEqual(expectObj, res.Object) {
t.Errorf("#%d: obj want=\n%#v\nget=\n%#v", i, expectObj, res.Object)
}
case <-time.After(wait.ForeverTestTimeout):
t.Errorf("#%d: time out after waiting %v on ResultChan", i, wait.ForeverTestTimeout)
}
}
func filterEvents(t *testing.T, ignoreBuilds map[string]struct{}, buildWatch watchapi.Interface) (newBuild *buildapi.Build, event watchapi.Event) {
for {
event = <-buildWatch.ResultChan()
var ok bool
newBuild, ok = event.Object.(*buildapi.Build)
if !ok {
t.Errorf("unexpected event type (not a Build): %v", event.Object)
}
if _, exists := ignoreBuilds[newBuild.Name]; !exists {
break
}
}
return
}
func waitForOnlyDelete(projectName string, w watch.Interface, t *testing.T) {
select {
case event := <-w.ResultChan():
project := event.Object.(*projectapi.Project)
t.Logf("got %#v %#v", event, project)
if event.Type == watch.Deleted && project.Name == projectName {
return
}
t.Errorf("got unexpected project %v", project.Name)
case <-time.After(30 * time.Second):
t.Fatalf("timeout: %v", projectName)
}
}
func waitForAdd(projectName string, w watch.Interface, t *testing.T) {
for {
select {
case event := <-w.ResultChan():
project := event.Object.(*projectapi.Project)
t.Logf("got %#v %#v", event, project)
if event.Type == watch.Added && project.Name == projectName {
return
}
case <-time.After(30 * time.Second):
t.Fatalf("timeout: %v", projectName)
}
}
}
// waitForWatchType tolerates receiving 3 events before failing while watching for a particular event
// type.
func waitForWatchType(t *testing.T, name string, w watchapi.Interface, expect watchapi.EventType) *watchapi.Event {
tries := 3
for i := 0; i < tries; i++ {
select {
case e := <-w.ResultChan():
if e.Type != expect {
continue
}
return &e
case <-time.After(BuildControllersWatchTimeout):
t.Fatalf("Timed out waiting for watch: %s", name)
return nil
}
}
t.Fatalf("Waited for a %v event with %d tries but never received one", expect, tries)
return nil
}
func testCheckStop(t *testing.T, i int, w watch.Interface) {
select {
case e, ok := <-w.ResultChan():
if ok {
var obj string
switch e.Object.(type) {
case *api.Pod:
obj = e.Object.(*api.Pod).Name
case *unversioned.Status:
obj = e.Object.(*unversioned.Status).Message
}
t.Errorf("#%d: ResultChan should have been closed. Event: %s. Object: %s", i, e.Type, obj)
}
case <-time.After(wait.ForeverTestTimeout):
t.Errorf("#%d: time out after waiting 1s on ResultChan", i)
}
}
func waitForPersistentVolumeClaimPhase(client *clientset.Clientset, claimName, namespace string, w watch.Interface, phase api.PersistentVolumeClaimPhase) {
// Check if the claim is already in requested phase
claim, err := client.Core().PersistentVolumeClaims(namespace).Get(claimName)
if err == nil && claim.Status.Phase == phase {
return
}
// Wait for the phase
for {
event := <-w.ResultChan()
claim, ok := event.Object.(*api.PersistentVolumeClaim)
if !ok {
continue
}
if claim.Status.Phase == phase && claim.Name == claimName {
glog.V(2).Infof("claim %q is %s", claim.Name, phase)
break
}
}
}
func waitForPersistentVolumePhase(client *clientset.Clientset, pvName string, w watch.Interface, phase api.PersistentVolumePhase) {
// Check if the volume is already in requested phase
volume, err := client.Core().PersistentVolumes().Get(pvName)
if err == nil && volume.Status.Phase == phase {
return
}
// Wait for the phase
for {
event := <-w.ResultChan()
volume, ok := event.Object.(*api.PersistentVolume)
if !ok {
continue
}
if volume.Status.Phase == phase && volume.Name == pvName {
glog.V(2).Infof("volume %q is %s", volume.Name, phase)
break
}
}
}
func observeObjectDeletion(w watch.Interface) (obj runtime.Object) {
deleted := false
timeout := false
timer := time.After(60 * time.Second)
for !deleted && !timeout {
select {
case event, _ := <-w.ResultChan():
if event.Type == watch.Deleted {
obj = event.Object
deleted = true
}
case <-timer:
timeout = true
}
}
if !deleted {
framework.Failf("Failed to observe pod deletion")
}
return
}
func observePodDeletion(w watch.Interface) (lastPod *api.Pod) {
deleted := false
timeout := false
timer := time.After(60 * time.Second)
for !deleted && !timeout {
select {
case event, _ := <-w.ResultChan():
if event.Type == watch.Deleted {
lastPod = event.Object.(*api.Pod)
deleted = true
}
case <-timer:
timeout = true
}
}
if !deleted {
Fail("Failed to observe pod deletion")
}
return
}
// ensure the watch delivers the requested and only the requested items.
func consume(t *testing.T, w watch.Interface, rvs []string, done *sync.WaitGroup) {
defer done.Done()
for _, rv := range rvs {
got, ok := <-w.ResultChan()
if !ok {
t.Errorf("%#v: unexpected channel close, wanted %v", rvs, rv)
return
}
gotRV := got.Object.(*api.Pod).ObjectMeta.ResourceVersion
if e, a := rv, gotRV; e != a {
t.Errorf("wanted %v, got %v", e, a)
} else {
t.Logf("Got %v as expected", gotRV)
}
}
// We should not get anything else.
got, open := <-w.ResultChan()
if open {
t.Errorf("%#v: unwanted object %#v", rvs, got)
}
}
func watcher(ctx context.Context) {
debugf("starting watcher\n")
var closed = watch.Event{}
var w watch.Interface
var wchan = make(chan watch.Interface)
defer close(wchan)
for {
go acquireWatch(ctx, wchan)
select {
case w = <-wchan:
debugf("watch acquired\n")
case <-ctx.Done():
debugf("watcher cancelled\n")
return
}
loop:
for {
select {
case e := <-w.ResultChan():
if e == closed {
debugf("watcher closed channel with event: %+v\n", e)
break loop
}
handleWatchEvent(e)
case <-ctx.Done():
w.Stop()
debugf("watcher cancelled\n")
return
}
}
}
}
// watchRollbackEvent watches for rollback events and returns rollback result
func watchRollbackEvent(w watch.Interface) string {
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt, os.Kill, syscall.SIGTERM)
for {
select {
case event, ok := <-w.ResultChan():
if !ok {
return ""
}
obj, ok := event.Object.(*api.Event)
if !ok {
w.Stop()
return ""
}
isRollback, result := isRollbackEvent(obj)
if isRollback {
w.Stop()
return result
}
case <-signals:
w.Stop()
}
}
}
// WatchLoop loops, passing events in w to fn.
// If user sends interrupt signal, shut down cleanly. Otherwise, never return.
func WatchLoop(w watch.Interface, fn func(watch.Event) error) {
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
defer signal.Stop(signals)
for {
select {
case event, ok := <-w.ResultChan():
if !ok {
return
}
if err := fn(event); err != nil {
w.Stop()
}
case <-signals:
w.Stop()
}
}
}
//
// Watch is the main entry-point for the service, we listen out for changes in the
// nodes, pods and the refresh timer
func (r *kubeAPIImpl) Watch(updates UpdateEvent) (ShutdownChannel, error) {
var err error
var nodeCh, podsCh watch.Interface
// step: create the done channel
shutdownCh := make(ShutdownChannel)
go func() {
// step: wait for the cleanup channel
<-shutdownCh
}()
// step: acquire a nodes watch
if nodeCh, err = r.createNodesWatch(); err != nil {
return nil, err
}
// step: create a channel for pod updates
if podsCh, err = r.createPodsWatch(); err != nil {
return nil, err
}
// notes: the main loop to the service; we wait for changes in the nodes,
// the pods or the refresh timer
go func() {
glog.V(10).Infof("Starting the event service loop")
for {
select {
case update := <-nodeCh.ResultChan():
// step: we only care about added or removed nodes, not modified
if update.Type == watch.Modified {
continue
}
event := newEvent(nodeEvent, update)
glog.V(5).Infof("Recieved an update to the nodes: %v", event)
updates <- event
case update := <-podsCh.ResultChan():
event := newEvent(podEvent, update)
glog.V(5).Infof("Recieved an update to the pods: %v", event)
updates <- event
}
}
}()
return shutdownCh, nil
}