// StartPods check for numPods in TestNS. If they exist, it no-ops, otherwise it starts up // a temp rc, scales it to match numPods, then deletes the rc leaving behind the pods. func StartPods(numPods int, host string, restClient *client.Client) error { start := time.Now() defer func() { glog.Infof("StartPods took %v with numPods %d", time.Since(start), numPods) }() hostField := fields.OneTermEqualSelector(client.PodHost, host) pods, err := restClient.Pods(TestNS).List(labels.Everything(), hostField) if err != nil || len(pods.Items) == numPods { return err } glog.Infof("Found %d pods that match host %v, require %d", len(pods.Items), hostField, numPods) // For the sake of simplicity, assume all pods in TestNS have selectors matching TestRCManifest. controller := RCFromManifest(TestRCManifest) // Make the rc unique to the given host. controller.Spec.Replicas = numPods controller.Spec.Template.Spec.NodeName = host controller.Name = controller.Name + host controller.Spec.Selector["host"] = host controller.Spec.Template.Labels["host"] = host if rc, err := StartRC(controller, restClient); err != nil { return err } else { // Delete the rc, otherwise when we restart master components for the next benchmark // the rc controller will race with the pods controller in the rc manager. return restClient.ReplicationControllers(TestNS).Delete(rc.Name) } }
// TestClusterPolicyListRespectingFields tests that a List() call, filtered with a field to the ReadOnlyClusterPolicyCache // will return all clusterPolicies matching that field func TestClusterPolicyListRespectingFields(t *testing.T) { testCache, cacheChannel, testChannel := beforeTestingSetup_readonlyclusterpolicycache() defer close(cacheChannel) var clusterPolicies *authorizationapi.ClusterPolicyList var err error name := "uniqueClusterPolicyName" label := labels.Everything() field := fields.OneTermEqualSelector("metadata.name", name) util.Until(func() { clusterPolicies, err = testCache.List(label, field) if (err == nil) && (clusterPolicies != nil) && (len(clusterPolicies.Items) == 1) && (clusterPolicies.Items[0].Name == name) { close(testChannel) } }, 1*time.Millisecond, testChannel) switch { case err != nil: t.Errorf("Error getting clusterPolicyList with fieldSelector using ReadOnlyClusterPolicyCache: %v", err) case clusterPolicies == nil: t.Error("ClusterPolicyList is nil.") case len(clusterPolicies.Items) != 1: t.Errorf("Expected clusterPolicyList to contain 2 clusterPolicies, contained %d", len(clusterPolicies.Items)) case clusterPolicies.Items[0].Name != name: t.Errorf("Expected field-selected clusterPolicy name to be '%s', was '%s'", name, clusterPolicies.Items[0].Name) } }
// TestPolicyListRespectingFields tests that a List() call for some namespace, filtered with a field to the ReadOnlyPolicyCache // will return all policies in that namespace matching that field func TestPolicyListRespectingFields(t *testing.T) { testCache, cacheChannel, testChannel := beforeTestingSetup_readonlypolicycache() defer close(cacheChannel) var policies *authorizationapi.PolicyList var err error name := "uniquePolicyName" namespace := "namespaceTwo" label := labels.Everything() field := fields.OneTermEqualSelector("metadata.name", name) util.Until(func() { policies, err = testCache.List(label, field, namespace) if (err == nil) && (policies != nil) && (len(policies.Items) == 1) && (policies.Items[0].Name == name) { close(testChannel) } }, 1*time.Millisecond, testChannel) switch { case err != nil: t.Errorf("Error getting policies using ReadOnlyPolicyCache: %v", err) case policies == nil: t.Error("PoliciesList is nil") case len(policies.Items) != 1: t.Errorf("Expected policyList to have 1 policy, had %d", len(policies.Items)) case policies.Items[0].Name != name: t.Errorf("Expected policy name to be '%s', was '%s'", name, policies.Items[0].Name) } }
// findDockercfgSecret checks all the secrets in the namespace to see if the token secret has any existing dockercfg secrets that reference it func (e *DockercfgTokenDeletedController) findDockercfgSecrets(tokenSecret *api.Secret) ([]*api.Secret, error) { dockercfgSecrets := []*api.Secret{} dockercfgSelector := fields.OneTermEqualSelector(client.SecretType, string(api.SecretTypeDockercfg)) potentialSecrets, err := e.client.Secrets(tokenSecret.Namespace).List(labels.Everything(), dockercfgSelector) if err != nil { return nil, err } for i, currSecret := range potentialSecrets.Items { if currSecret.Annotations[ServiceAccountTokenSecretNameKey] == tokenSecret.Name { dockercfgSecrets = append(dockercfgSecrets, &potentialSecrets.Items[i]) } } return dockercfgSecrets, nil }
// deletePods will delete all pods from master running on given node. func (nc *NodeController) deletePods(nodeID string) error { glog.V(2).Infof("Delete all pods from %v", nodeID) pods, err := nc.kubeClient.Pods(api.NamespaceAll).List(labels.Everything(), fields.OneTermEqualSelector(client.PodHost, nodeID)) if err != nil { return err } for _, pod := range pods.Items { // Defensive check, also needed for tests. if pod.Spec.NodeName != nodeID { continue } glog.V(2).Infof("Delete pod %v", pod.Name) if err := nc.kubeClient.Pods(pod.Namespace).Delete(pod.Name, nil); err != nil { glog.Errorf("Error deleting pod %v: %v", pod.Name, err) } } return nil }
// Benchmark pod listing by waiting on `Tasks` listers to list `Pods` pods via `Workers`. func BenchmarkPodList(b *testing.B) { b.StopTimer() m := framework.NewMasterComponents(&framework.Config{nil, true, false, 250.0, 500}) defer m.Stop(true, true) numPods, numTasks, iter := getPods(b.N), getTasks(b.N), getIterations(b.N) podsPerNode := numPods / numTasks if podsPerNode < 1 { podsPerNode = 1 } glog.Infof("Starting benchmark: b.N %d, pods %d, workers %d, podsPerNode %d", b.N, numPods, numTasks, podsPerNode) startPodsOnNodes(numPods, numTasks, m.RestClient) // Stop the rc manager so it doesn't steal resources m.Stop(false, true) b.StartTimer() for i := 0; i < iter; i++ { framework.RunParallel(func(id int) error { host := fmt.Sprintf("host.%d", id) now := time.Now() defer func() { glog.V(3).Infof("Worker %d: Node %v listing pods took %v", id, host, time.Since(now)) }() if pods, err := m.RestClient.Pods(framework.TestNS).List( labels.Everything(), fields.OneTermEqualSelector(client.PodHost, host)); err != nil { return err } else if len(pods.Items) < podsPerNode { glog.Fatalf("List retrieved %d pods, which is less than %d", len(pods.Items), podsPerNode) } return nil }, numTasks, Workers) } b.StopTimer() }
// NewDockercfgTokenDeletedController returns a new *DockercfgTokenDeletedController. func NewDockercfgTokenDeletedController(cl client.Interface, options DockercfgTokenDeletedControllerOptions) *DockercfgTokenDeletedController { e := &DockercfgTokenDeletedController{ client: cl, } dockercfgSelector := fields.OneTermEqualSelector(client.SecretType, string(api.SecretTypeServiceAccountToken)) _, e.secretController = framework.NewInformer( &cache.ListWatch{ ListFunc: func() (runtime.Object, error) { return e.client.Secrets(api.NamespaceAll).List(labels.Everything(), dockercfgSelector) }, WatchFunc: func(rv string) (watch.Interface, error) { return e.client.Secrets(api.NamespaceAll).Watch(labels.Everything(), dockercfgSelector, rv) }, }, &api.Secret{}, options.Resync, framework.ResourceEventHandlerFuncs{ DeleteFunc: e.secretDeleted, }, ) return e }
// NewSourceApiserver creates a config source that watches and pulls from the apiserver. func NewSourceApiserver(c *client.Client, hostname string, updates chan<- interface{}) { lw := cache.NewListWatchFromClient(c, "pods", api.NamespaceAll, fields.OneTermEqualSelector(client.PodHost, hostname)) newSourceApiserverFromLW(lw, updates) }
// ListResource returns a function that handles retrieving a list of resources from a rest.Storage object. func ListResource(r rest.Lister, rw rest.Watcher, scope RequestScope, forceWatch bool) restful.RouteFunction { return func(req *restful.Request, res *restful.Response) { w := res.ResponseWriter namespace, err := scope.Namer.Namespace(req) if err != nil { errorJSON(err, scope.Codec, w) return } // Watches for single objects are routed to this function. // Treat a /name parameter the same as a field selector entry. hasName := true _, name, err := scope.Namer.Name(req) if err != nil { hasName = false } ctx := scope.ContextFunc(req) ctx = api.WithNamespace(ctx, namespace) out, err := queryToObject(req.Request.URL.Query(), scope, "ListOptions") if err != nil { errorJSON(err, scope.Codec, w) return } opts := *out.(*api.ListOptions) // transform fields // TODO: queryToObject should do this. fn := func(label, value string) (newLabel, newValue string, err error) { return scope.Convertor.ConvertFieldLabel(scope.APIVersion, scope.Kind, label, value) } if opts.FieldSelector, err = opts.FieldSelector.Transform(fn); err != nil { // TODO: allow bad request to set field causes based on query parameters err = errors.NewBadRequest(err.Error()) errorJSON(err, scope.Codec, w) return } if hasName { // metadata.name is the canonical internal name. // generic.SelectionPredicate will notice that this is // a request for a single object and optimize the // storage query accordingly. nameSelector := fields.OneTermEqualSelector("metadata.name", name) if opts.FieldSelector != nil && !opts.FieldSelector.Empty() { // It doesn't make sense to ask for both a name // and a field selector, since just the name is // sufficient to narrow down the request to a // single object. errorJSON( errors.NewBadRequest("both a name and a field selector provided; please provide one or the other."), scope.Codec, w, ) return } opts.FieldSelector = nameSelector } if (opts.Watch || forceWatch) && rw != nil { watcher, err := rw.Watch(ctx, opts.LabelSelector, opts.FieldSelector, opts.ResourceVersion) if err != nil { errorJSON(err, scope.Codec, w) return } serveWatch(watcher, scope, w, req) return } result, err := r.List(ctx, opts.LabelSelector, opts.FieldSelector) if err != nil { errorJSON(err, scope.Codec, w) return } if err := setListSelfLink(result, req, scope.Namer); err != nil { errorJSON(err, scope.Codec, w) return } write(http.StatusOK, scope.APIVersion, scope.Codec, result, w, req.Request) } }
// TestConcurrentBuildControllers tests the transition of a build from new to pending. Ensures that only a single New -> Pending // transition happens and that only a single pod is created during a set period of time. func TestConcurrentBuildControllers(t *testing.T) { // Start a master with multiple BuildControllers osClient, kClient := setupBuildControllerTest(5, 0, 0, t) // Setup an error channel errChan := make(chan error) // go routines will send a message on this channel if an error occurs. Once this happens the test is over // Create a build ns := testutil.Namespace() b, err := osClient.Builds(ns).Create(mockBuild()) checkErr(t, err) // Start watching builds for New -> Pending transition buildWatch, err := osClient.Builds(ns).Watch(labels.Everything(), fields.OneTermEqualSelector("name", b.Name), b.ResourceVersion) checkErr(t, err) defer buildWatch.Stop() buildModifiedCount := int32(0) go func() { for e := range buildWatch.ResultChan() { if e.Type != watchapi.Modified { errChan <- fmt.Errorf("received an unexpected event of type: %s with object: %#v", e.Type, e.Object) } build, ok := e.Object.(*buildapi.Build) if !ok { errChan <- fmt.Errorf("received something other than build: %#v", e.Object) break } // If unexpected status, throw error if build.Status.Phase != buildapi.BuildPhasePending { errChan <- fmt.Errorf("received unexpected build status: %s", build.Status.Phase) break } else { atomic.AddInt32(&buildModifiedCount, 1) } } }() // Watch build pods as they are created podWatch, err := kClient.Pods(ns).Watch(labels.Everything(), fields.OneTermEqualSelector("metadata.name", buildutil.GetBuildPodName(b)), "") checkErr(t, err) defer podWatch.Stop() podAddedCount := int32(0) go func() { for e := range podWatch.ResultChan() { // Look for creation events if e.Type == watchapi.Added { atomic.AddInt32(&podAddedCount, 1) } } }() select { case err := <-errChan: t.Errorf("Error: %v", err) case <-time.After(ConcurrentBuildControllersTestWait): if atomic.LoadInt32(&buildModifiedCount) != 1 { t.Errorf("The build was modified an unexpected number of times. Got: %d, Expected: 1", buildModifiedCount) } if atomic.LoadInt32(&podAddedCount) != 1 { t.Errorf("The build pod was created an unexpected number of times. Got: %d, Expected: 1", podAddedCount) } } }
func runImageChangeTriggerTest(t *testing.T, clusterAdminClient *client.Client, imageStream *imageapi.ImageStream, imageStreamMapping *imageapi.ImageStreamMapping, config *buildapi.BuildConfig, tag string) { created, err := clusterAdminClient.BuildConfigs(testutil.Namespace()).Create(config) if err != nil { t.Fatalf("Couldn't create BuildConfig: %v", err) } watch, err := clusterAdminClient.Builds(testutil.Namespace()).Watch(labels.Everything(), fields.Everything(), created.ResourceVersion) if err != nil { t.Fatalf("Couldn't subscribe to Builds %v", err) } watch2, err := clusterAdminClient.BuildConfigs(testutil.Namespace()).Watch(labels.Everything(), fields.Everything(), created.ResourceVersion) if err != nil { t.Fatalf("Couldn't subscribe to BuildConfigs %v", err) } defer watch2.Stop() imageStream, err = clusterAdminClient.ImageStreams(testutil.Namespace()).Create(imageStream) if err != nil { t.Fatalf("Couldn't create ImageStream: %v", err) } err = clusterAdminClient.ImageStreamMappings(testutil.Namespace()).Create(imageStreamMapping) if err != nil { t.Fatalf("Couldn't create Image: %v", err) } // wait for initial build event from the creation of the imagerepo with tag latest event := waitForWatch(t, "initial build added", watch) if e, a := watchapi.Added, event.Type; e != a { t.Fatalf("expected watch event type %s, got %s", e, a) } newBuild := event.Object.(*buildapi.Build) switch newBuild.Spec.Strategy.Type { case buildapi.SourceBuildStrategyType: if newBuild.Spec.Strategy.SourceStrategy.From.Name != "registry:8080/openshift/test-image-trigger:"+tag { i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name) bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name) t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\ntrigger is %s\n", "registry:8080/openshift/test-image-trigger:"+tag, newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[0].ImageChange) } case buildapi.DockerBuildStrategyType: if newBuild.Spec.Strategy.DockerStrategy.From.Name != "registry:8080/openshift/test-image-trigger:"+tag { i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name) bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name) t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\ntrigger is %s\n", "registry:8080/openshift/test-image-trigger:"+tag, newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[0].ImageChange) } case buildapi.CustomBuildStrategyType: if newBuild.Spec.Strategy.CustomStrategy.From.Name != "registry:8080/openshift/test-image-trigger:"+tag { i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name) bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name) t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\ntrigger is %s\n", "registry:8080/openshift/test-image-trigger:"+tag, newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[0].ImageChange) } } // Wait for an update on the specific build that was added watch3, err := clusterAdminClient.Builds(testutil.Namespace()).Watch(labels.Everything(), fields.OneTermEqualSelector("name", newBuild.Name), newBuild.ResourceVersion) defer watch3.Stop() if err != nil { t.Fatalf("Couldn't subscribe to Builds %v", err) } event = waitForWatch(t, "initial build update", watch3) if e, a := watchapi.Modified, event.Type; e != a { t.Fatalf("expected watch event type %s, got %s", e, a) } newBuild = event.Object.(*buildapi.Build) // Make sure the resolution of the build's docker image pushspec didn't mutate the persisted API object if newBuild.Spec.Output.To.Name != "test-image-trigger-repo:outputtag" { t.Fatalf("unexpected build output: %#v %#v", newBuild.Spec.Output.To, newBuild.Spec.Output) } if newBuild.Labels["testlabel"] != "testvalue" { t.Fatalf("Expected build with label %s=%s from build config got %s=%s", "testlabel", "testvalue", "testlabel", newBuild.Labels["testlabel"]) } // wait for build config to be updated WaitLoop: for { select { case e := <-watch2.ResultChan(): event = &e continue case <-time.After(BuildControllersWatchTimeout): break WaitLoop } } updatedConfig := event.Object.(*buildapi.BuildConfig) if err != nil { t.Fatalf("Couldn't get BuildConfig: %v", err) } // the first tag did not have an image id, so the last trigger field is the pull spec if updatedConfig.Spec.Triggers[0].ImageChange.LastTriggeredImageID != "registry:8080/openshift/test-image-trigger:"+tag { t.Fatalf("Expected imageID equal to pull spec, got %#v", updatedConfig.Spec.Triggers[0].ImageChange) } // clear out the build/buildconfig watches before triggering a new build WaitLoop2: for { select { case <-watch.ResultChan(): continue case <-watch2.ResultChan(): continue case <-time.After(BuildControllersWatchTimeout): break WaitLoop2 } } // trigger a build by posting a new image if err := clusterAdminClient.ImageStreamMappings(testutil.Namespace()).Create(&imageapi.ImageStreamMapping{ ObjectMeta: kapi.ObjectMeta{ Namespace: testutil.Namespace(), Name: imageStream.Name, }, Tag: tag, Image: imageapi.Image{ ObjectMeta: kapi.ObjectMeta{ Name: "ref-2-random", }, DockerImageReference: "registry:8080/openshift/test-image-trigger:ref-2-random", }, }); err != nil { t.Fatalf("unexpected error: %v", err) } event = waitForWatch(t, "second build created", watch) if e, a := watchapi.Added, event.Type; e != a { t.Fatalf("expected watch event type %s, got %s", e, a) } newBuild = event.Object.(*buildapi.Build) switch newBuild.Spec.Strategy.Type { case buildapi.SourceBuildStrategyType: if newBuild.Spec.Strategy.SourceStrategy.From.Name != "registry:8080/openshift/test-image-trigger:ref-2-random" { i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name) bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name) t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\trigger is %s\n", "registry:8080/openshift/test-image-trigger:ref-2-random", newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[3].ImageChange) } case buildapi.DockerBuildStrategyType: if newBuild.Spec.Strategy.DockerStrategy.From.Name != "registry:8080/openshift/test-image-trigger:ref-2-random" { i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name) bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name) t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\trigger is %s\n", "registry:8080/openshift/test-image-trigger:ref-2-random", newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[3].ImageChange) } case buildapi.CustomBuildStrategyType: if newBuild.Spec.Strategy.CustomStrategy.From.Name != "registry:8080/openshift/test-image-trigger:ref-2-random" { i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name) bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name) t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\trigger is %s\n", "registry:8080/openshift/test-image-trigger:ref-2-random", newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[3].ImageChange) } } // Listen to events on specific build watch4, err := clusterAdminClient.Builds(testutil.Namespace()).Watch(labels.Everything(), fields.OneTermEqualSelector("name", newBuild.Name), newBuild.ResourceVersion) defer watch4.Stop() event = waitForWatch(t, "update on second build", watch4) if e, a := watchapi.Modified, event.Type; e != a { t.Fatalf("expected watch event type %s, got %s", e, a) } newBuild = event.Object.(*buildapi.Build) // Make sure the resolution of the build's docker image pushspec didn't mutate the persisted API object if newBuild.Spec.Output.To.Name != "test-image-trigger-repo:outputtag" { t.Fatalf("unexpected build output: %#v %#v", newBuild.Spec.Output.To, newBuild.Spec.Output) } if newBuild.Labels["testlabel"] != "testvalue" { t.Fatalf("Expected build with label %s=%s from build config got %s=%s", "testlabel", "testvalue", "testlabel", newBuild.Labels["testlabel"]) } WaitLoop3: for { select { case e := <-watch2.ResultChan(): event = &e continue case <-time.After(BuildControllersWatchTimeout): break WaitLoop3 } } updatedConfig = event.Object.(*buildapi.BuildConfig) if e, a := "registry:8080/openshift/test-image-trigger:ref-2-random", updatedConfig.Spec.Triggers[0].ImageChange.LastTriggeredImageID; e != a { t.Errorf("unexpected trigger id: expected %v, got %v", e, a) } }
// TestConcurrentBuildPodControllers tests the lifecycle of a build pod when running multiple controllers. func TestConcurrentBuildPodControllers(t *testing.T) { // Start a master with multiple BuildPodControllers osClient, kClient := setupBuildControllerTest(0, 5, 0, t) ns := testutil.Namespace() waitTime := ConcurrentBuildPodControllersTestWait tests := []buildControllerPodTest{ { Name: "running state test", States: []buildControllerPodState{ { PodPhase: kapi.PodRunning, BuildPhase: buildapi.BuildPhaseRunning, }, }, }, { Name: "build succeeded", States: []buildControllerPodState{ { PodPhase: kapi.PodRunning, BuildPhase: buildapi.BuildPhaseRunning, }, { PodPhase: kapi.PodSucceeded, BuildPhase: buildapi.BuildPhaseComplete, }, }, }, { Name: "build failed", States: []buildControllerPodState{ { PodPhase: kapi.PodRunning, BuildPhase: buildapi.BuildPhaseRunning, }, { PodPhase: kapi.PodFailed, BuildPhase: buildapi.BuildPhaseFailed, }, }, }, } for _, test := range tests { // Setup communications channels podReadyChan := make(chan *kapi.Pod) // Will receive a value when a build pod is ready errChan := make(chan error) // Will receive a value when an error occurs stateReached := int32(0) // Create a build b, err := osClient.Builds(ns).Create(mockBuild()) checkErr(t, err) // Watch build pod for transition to pending podWatch, err := kClient.Pods(ns).Watch(labels.Everything(), fields.OneTermEqualSelector("metadata.name", buildutil.GetBuildPodName(b)), "") checkErr(t, err) go func() { for e := range podWatch.ResultChan() { pod, ok := e.Object.(*kapi.Pod) if !ok { checkErr(t, fmt.Errorf("%s: unexpected object received: %#v\n", test.Name, e.Object)) } if pod.Status.Phase == kapi.PodPending { podReadyChan <- pod break } } }() var pod *kapi.Pod select { case pod = <-podReadyChan: if pod.Status.Phase != kapi.PodPending { t.Errorf("Got wrong pod phase: %s", pod.Status.Phase) podWatch.Stop() continue } case <-time.After(BuildControllersWatchTimeout): t.Errorf("Timed out waiting for build pod to be ready") podWatch.Stop() continue } podWatch.Stop() for _, state := range test.States { // Update pod state and verify that corresponding build state happens accordingly pod, err := kClient.Pods(ns).Get(pod.Name) checkErr(t, err) pod.Status.Phase = state.PodPhase _, err = kClient.Pods(ns).UpdateStatus(pod) checkErr(t, err) buildWatch, err := osClient.Builds(ns).Watch(labels.Everything(), fields.OneTermEqualSelector("name", b.Name), b.ResourceVersion) checkErr(t, err) defer buildWatch.Stop() go func() { done := false for e := range buildWatch.ResultChan() { var ok bool b, ok = e.Object.(*buildapi.Build) if !ok { errChan <- fmt.Errorf("%s: unexpected object received: %#v", test.Name, e.Object) } if e.Type != watchapi.Modified { errChan <- fmt.Errorf("%s: unexpected event received: %s, object: %#v", e.Type, e.Object) } if done { errChan <- fmt.Errorf("%s: unexpected build state: %#v", test.Name, e.Object) } else if b.Status.Phase == state.BuildPhase { done = true atomic.StoreInt32(&stateReached, 1) } } }() select { case err := <-errChan: buildWatch.Stop() t.Errorf("Error: %v\n", test.Name, err) break case <-time.After(waitTime): buildWatch.Stop() if atomic.LoadInt32(&stateReached) != 1 { t.Errorf("%s: Did not reach desired build state: %s", test.Name, state.BuildPhase) break } } } } }
// rebootNode takes node name on provider through the following steps using c: // - ensures the node is ready // - ensures all pods on the node are running and ready // - reboots the node (by executing rebootCmd over ssh) // - ensures the node reaches some non-ready state // - ensures the node becomes ready again // - ensures all pods on the node become running and ready again // // It returns true through result only if all of the steps pass; at the first // failed step, it will return false through result and not run the rest. func rebootNode(c *client.Client, provider, name, rebootCmd string, result chan bool) { // Setup ns := api.NamespaceDefault ps := newPodStore(c, ns, labels.Everything(), fields.OneTermEqualSelector(client.PodHost, name)) defer ps.Stop() // Get the node initially. Logf("Getting %s", name) node, err := c.Nodes().Get(name) if err != nil { Logf("Couldn't get node %s", name) result <- false return } // Node sanity check: ensure it is "ready". if !waitForNodeToBeReady(c, name, nodeReadyInitialTimeout) { result <- false return } // Get all the pods on the node. pods := ps.List() podNames := make([]string, len(pods)) for i, p := range pods { podNames[i] = p.ObjectMeta.Name } Logf("Node %s has %d pods: %v", name, len(podNames), podNames) // For each pod, we do a sanity check to ensure it's running / healthy // now, as that's what we'll be checking later. if !checkPodsRunningReady(c, ns, podNames, podReadyBeforeTimeout) { result <- false return } // Reboot the node. if err = issueSSHCommand(node, provider, rebootCmd); err != nil { Logf("Error while issuing ssh command: %v", err) result <- false return } // Wait for some kind of "not ready" status. if !waitForNodeToBeNotReady(c, name, rebootNodeNotReadyTimeout) { result <- false return } // Wait for some kind of "ready" status. if !waitForNodeToBeReady(c, name, rebootNodeReadyAgainTimeout) { result <- false return } // Ensure all of the pods that we found on this node before the reboot are // running / healthy. if !checkPodsRunningReady(c, ns, podNames, rebootPodReadyAgainTimeout) { result <- false return } Logf("Reboot successful on node %s", name) result <- true }
// rebootNode takes node name on provider through the following steps using c: // - ensures the node is ready // - ensures all pods on the node are running and ready // - reboots the node (by executing rebootCmd over ssh) // - ensures the node reaches some non-ready state // - ensures the node becomes ready again // - ensures all pods on the node become running and ready again // // It returns true through result only if all of the steps pass; at the first // failed step, it will return false through result and not run the rest. func rebootNode(c *client.Client, provider, name, rebootCmd string, result chan bool) { // Setup ns := api.NamespaceSystem ps := newPodStore(c, ns, labels.Everything(), fields.OneTermEqualSelector(client.PodHost, name)) defer ps.Stop() // Get the node initially. Logf("Getting %s", name) node, err := c.Nodes().Get(name) if err != nil { Logf("Couldn't get node %s", name) result <- false return } // Node sanity check: ensure it is "ready". if !waitForNodeToBeReady(c, name, nodeReadyInitialTimeout) { result <- false return } // Get all the pods on the node that don't have liveness probe set. // Liveness probe may cause restart of a pod during node reboot, and the pod may not be running. pods := ps.List() podNames := []string{} for _, p := range pods { probe := false for _, c := range p.Spec.Containers { if c.LivenessProbe != nil { probe = true break } } if !probe { podNames = append(podNames, p.ObjectMeta.Name) } } Logf("Node %s has %d pods: %v", name, len(podNames), podNames) // For each pod, we do a sanity check to ensure it's running / healthy // now, as that's what we'll be checking later. if !checkPodsRunningReady(c, ns, podNames, podReadyBeforeTimeout) { result <- false return } // Reboot the node. if err = issueSSHCommand(node, provider, rebootCmd); err != nil { Logf("Error while issuing ssh command: %v", err) result <- false return } // Wait for some kind of "not ready" status. if !waitForNodeToBeNotReady(c, name, rebootNodeNotReadyTimeout) { result <- false return } // Wait for some kind of "ready" status. if !waitForNodeToBeReady(c, name, rebootNodeReadyAgainTimeout) { result <- false return } // Ensure all of the pods that we found on this node before the reboot are // running / healthy. if !checkPodsRunningReady(c, ns, podNames, rebootPodReadyAgainTimeout) { result <- false return } Logf("Reboot successful on node %s", name) result <- true }
// NewSourceApiserver creates a config source that watches and pulls from the apiserver. func NewSourceApiserver(client *client.Client, hostname string, updates chan<- interface{}) { lw := cache.NewListWatchFromClient(client, "pods", api.NamespaceAll, fields.OneTermEqualSelector(getHostFieldLabel(client.APIVersion()), hostname)) newSourceApiserverFromLW(lw, updates) }