// StartPods check for numPods in TestNS. If they exist, it no-ops, otherwise it starts up
// a temp rc, scales it to match numPods, then deletes the rc leaving behind the pods.
func StartPods(numPods int, host string, restClient *client.Client) error {
start := time.Now()
defer func() {
glog.Infof("StartPods took %v with numPods %d", time.Since(start), numPods)
}()
hostField := fields.OneTermEqualSelector(client.PodHost, host)
pods, err := restClient.Pods(TestNS).List(labels.Everything(), hostField)
if err != nil || len(pods.Items) == numPods {
return err
}
glog.Infof("Found %d pods that match host %v, require %d", len(pods.Items), hostField, numPods)
// For the sake of simplicity, assume all pods in TestNS have selectors matching TestRCManifest.
controller := RCFromManifest(TestRCManifest)
// Make the rc unique to the given host.
controller.Spec.Replicas = numPods
controller.Spec.Template.Spec.NodeName = host
controller.Name = controller.Name + host
controller.Spec.Selector["host"] = host
controller.Spec.Template.Labels["host"] = host
if rc, err := StartRC(controller, restClient); err != nil {
return err
} else {
// Delete the rc, otherwise when we restart master components for the next benchmark
// the rc controller will race with the pods controller in the rc manager.
return restClient.ReplicationControllers(TestNS).Delete(rc.Name)
}
}
// TestClusterPolicyListRespectingFields tests that a List() call, filtered with a field to the ReadOnlyClusterPolicyCache
// will return all clusterPolicies matching that field
func TestClusterPolicyListRespectingFields(t *testing.T) {
testCache, cacheChannel, testChannel := beforeTestingSetup_readonlyclusterpolicycache()
defer close(cacheChannel)
var clusterPolicies *authorizationapi.ClusterPolicyList
var err error
name := "uniqueClusterPolicyName"
label := labels.Everything()
field := fields.OneTermEqualSelector("metadata.name", name)
util.Until(func() {
clusterPolicies, err = testCache.List(label, field)
if (err == nil) &&
(clusterPolicies != nil) &&
(len(clusterPolicies.Items) == 1) &&
(clusterPolicies.Items[0].Name == name) {
close(testChannel)
}
}, 1*time.Millisecond, testChannel)
switch {
case err != nil:
t.Errorf("Error getting clusterPolicyList with fieldSelector using ReadOnlyClusterPolicyCache: %v", err)
case clusterPolicies == nil:
t.Error("ClusterPolicyList is nil.")
case len(clusterPolicies.Items) != 1:
t.Errorf("Expected clusterPolicyList to contain 2 clusterPolicies, contained %d", len(clusterPolicies.Items))
case clusterPolicies.Items[0].Name != name:
t.Errorf("Expected field-selected clusterPolicy name to be '%s', was '%s'", name, clusterPolicies.Items[0].Name)
}
}
// TestPolicyListRespectingFields tests that a List() call for some namespace, filtered with a field to the ReadOnlyPolicyCache
// will return all policies in that namespace matching that field
func TestPolicyListRespectingFields(t *testing.T) {
testCache, cacheChannel, testChannel := beforeTestingSetup_readonlypolicycache()
defer close(cacheChannel)
var policies *authorizationapi.PolicyList
var err error
name := "uniquePolicyName"
namespace := "namespaceTwo"
label := labels.Everything()
field := fields.OneTermEqualSelector("metadata.name", name)
util.Until(func() {
policies, err = testCache.List(label, field, namespace)
if (err == nil) &&
(policies != nil) &&
(len(policies.Items) == 1) &&
(policies.Items[0].Name == name) {
close(testChannel)
}
}, 1*time.Millisecond, testChannel)
switch {
case err != nil:
t.Errorf("Error getting policies using ReadOnlyPolicyCache: %v", err)
case policies == nil:
t.Error("PoliciesList is nil")
case len(policies.Items) != 1:
t.Errorf("Expected policyList to have 1 policy, had %d", len(policies.Items))
case policies.Items[0].Name != name:
t.Errorf("Expected policy name to be '%s', was '%s'", name, policies.Items[0].Name)
}
}
// findDockercfgSecret checks all the secrets in the namespace to see if the token secret has any existing dockercfg secrets that reference it
func (e *DockercfgTokenDeletedController) findDockercfgSecrets(tokenSecret *api.Secret) ([]*api.Secret, error) {
dockercfgSecrets := []*api.Secret{}
dockercfgSelector := fields.OneTermEqualSelector(client.SecretType, string(api.SecretTypeDockercfg))
potentialSecrets, err := e.client.Secrets(tokenSecret.Namespace).List(labels.Everything(), dockercfgSelector)
if err != nil {
return nil, err
}
for i, currSecret := range potentialSecrets.Items {
if currSecret.Annotations[ServiceAccountTokenSecretNameKey] == tokenSecret.Name {
dockercfgSecrets = append(dockercfgSecrets, &potentialSecrets.Items[i])
}
}
return dockercfgSecrets, nil
}
// deletePods will delete all pods from master running on given node.
func (nc *NodeController) deletePods(nodeID string) error {
glog.V(2).Infof("Delete all pods from %v", nodeID)
pods, err := nc.kubeClient.Pods(api.NamespaceAll).List(labels.Everything(),
fields.OneTermEqualSelector(client.PodHost, nodeID))
if err != nil {
return err
}
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeID {
continue
}
glog.V(2).Infof("Delete pod %v", pod.Name)
if err := nc.kubeClient.Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
glog.Errorf("Error deleting pod %v: %v", pod.Name, err)
}
}
return nil
}
// Benchmark pod listing by waiting on `Tasks` listers to list `Pods` pods via `Workers`.
func BenchmarkPodList(b *testing.B) {
b.StopTimer()
m := framework.NewMasterComponents(&framework.Config{nil, true, false, 250.0, 500})
defer m.Stop(true, true)
numPods, numTasks, iter := getPods(b.N), getTasks(b.N), getIterations(b.N)
podsPerNode := numPods / numTasks
if podsPerNode < 1 {
podsPerNode = 1
}
glog.Infof("Starting benchmark: b.N %d, pods %d, workers %d, podsPerNode %d",
b.N, numPods, numTasks, podsPerNode)
startPodsOnNodes(numPods, numTasks, m.RestClient)
// Stop the rc manager so it doesn't steal resources
m.Stop(false, true)
b.StartTimer()
for i := 0; i < iter; i++ {
framework.RunParallel(func(id int) error {
host := fmt.Sprintf("host.%d", id)
now := time.Now()
defer func() {
glog.V(3).Infof("Worker %d: Node %v listing pods took %v", id, host, time.Since(now))
}()
if pods, err := m.RestClient.Pods(framework.TestNS).List(
labels.Everything(),
fields.OneTermEqualSelector(client.PodHost, host)); err != nil {
return err
} else if len(pods.Items) < podsPerNode {
glog.Fatalf("List retrieved %d pods, which is less than %d", len(pods.Items), podsPerNode)
}
return nil
}, numTasks, Workers)
}
b.StopTimer()
}
// NewDockercfgTokenDeletedController returns a new *DockercfgTokenDeletedController.
func NewDockercfgTokenDeletedController(cl client.Interface, options DockercfgTokenDeletedControllerOptions) *DockercfgTokenDeletedController {
e := &DockercfgTokenDeletedController{
client: cl,
}
dockercfgSelector := fields.OneTermEqualSelector(client.SecretType, string(api.SecretTypeServiceAccountToken))
_, e.secretController = framework.NewInformer(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return e.client.Secrets(api.NamespaceAll).List(labels.Everything(), dockercfgSelector)
},
WatchFunc: func(rv string) (watch.Interface, error) {
return e.client.Secrets(api.NamespaceAll).Watch(labels.Everything(), dockercfgSelector, rv)
},
},
&api.Secret{},
options.Resync,
framework.ResourceEventHandlerFuncs{
DeleteFunc: e.secretDeleted,
},
)
return e
}
// NewSourceApiserver creates a config source that watches and pulls from the apiserver.
func NewSourceApiserver(c *client.Client, hostname string, updates chan<- interface{}) {
lw := cache.NewListWatchFromClient(c, "pods", api.NamespaceAll, fields.OneTermEqualSelector(client.PodHost, hostname))
newSourceApiserverFromLW(lw, updates)
}
// ListResource returns a function that handles retrieving a list of resources from a rest.Storage object.
func ListResource(r rest.Lister, rw rest.Watcher, scope RequestScope, forceWatch bool) restful.RouteFunction {
return func(req *restful.Request, res *restful.Response) {
w := res.ResponseWriter
namespace, err := scope.Namer.Namespace(req)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
// Watches for single objects are routed to this function.
// Treat a /name parameter the same as a field selector entry.
hasName := true
_, name, err := scope.Namer.Name(req)
if err != nil {
hasName = false
}
ctx := scope.ContextFunc(req)
ctx = api.WithNamespace(ctx, namespace)
out, err := queryToObject(req.Request.URL.Query(), scope, "ListOptions")
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
opts := *out.(*api.ListOptions)
// transform fields
// TODO: queryToObject should do this.
fn := func(label, value string) (newLabel, newValue string, err error) {
return scope.Convertor.ConvertFieldLabel(scope.APIVersion, scope.Kind, label, value)
}
if opts.FieldSelector, err = opts.FieldSelector.Transform(fn); err != nil {
// TODO: allow bad request to set field causes based on query parameters
err = errors.NewBadRequest(err.Error())
errorJSON(err, scope.Codec, w)
return
}
if hasName {
// metadata.name is the canonical internal name.
// generic.SelectionPredicate will notice that this is
// a request for a single object and optimize the
// storage query accordingly.
nameSelector := fields.OneTermEqualSelector("metadata.name", name)
if opts.FieldSelector != nil && !opts.FieldSelector.Empty() {
// It doesn't make sense to ask for both a name
// and a field selector, since just the name is
// sufficient to narrow down the request to a
// single object.
errorJSON(
errors.NewBadRequest("both a name and a field selector provided; please provide one or the other."),
scope.Codec,
w,
)
return
}
opts.FieldSelector = nameSelector
}
if (opts.Watch || forceWatch) && rw != nil {
watcher, err := rw.Watch(ctx, opts.LabelSelector, opts.FieldSelector, opts.ResourceVersion)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
serveWatch(watcher, scope, w, req)
return
}
result, err := r.List(ctx, opts.LabelSelector, opts.FieldSelector)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if err := setListSelfLink(result, req, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
write(http.StatusOK, scope.APIVersion, scope.Codec, result, w, req.Request)
}
}
// TestConcurrentBuildControllers tests the transition of a build from new to pending. Ensures that only a single New -> Pending
// transition happens and that only a single pod is created during a set period of time.
func TestConcurrentBuildControllers(t *testing.T) {
// Start a master with multiple BuildControllers
osClient, kClient := setupBuildControllerTest(5, 0, 0, t)
// Setup an error channel
errChan := make(chan error) // go routines will send a message on this channel if an error occurs. Once this happens the test is over
// Create a build
ns := testutil.Namespace()
b, err := osClient.Builds(ns).Create(mockBuild())
checkErr(t, err)
// Start watching builds for New -> Pending transition
buildWatch, err := osClient.Builds(ns).Watch(labels.Everything(), fields.OneTermEqualSelector("name", b.Name), b.ResourceVersion)
checkErr(t, err)
defer buildWatch.Stop()
buildModifiedCount := int32(0)
go func() {
for e := range buildWatch.ResultChan() {
if e.Type != watchapi.Modified {
errChan <- fmt.Errorf("received an unexpected event of type: %s with object: %#v", e.Type, e.Object)
}
build, ok := e.Object.(*buildapi.Build)
if !ok {
errChan <- fmt.Errorf("received something other than build: %#v", e.Object)
break
}
// If unexpected status, throw error
if build.Status.Phase != buildapi.BuildPhasePending {
errChan <- fmt.Errorf("received unexpected build status: %s", build.Status.Phase)
break
} else {
atomic.AddInt32(&buildModifiedCount, 1)
}
}
}()
// Watch build pods as they are created
podWatch, err := kClient.Pods(ns).Watch(labels.Everything(), fields.OneTermEqualSelector("metadata.name", buildutil.GetBuildPodName(b)), "")
checkErr(t, err)
defer podWatch.Stop()
podAddedCount := int32(0)
go func() {
for e := range podWatch.ResultChan() {
// Look for creation events
if e.Type == watchapi.Added {
atomic.AddInt32(&podAddedCount, 1)
}
}
}()
select {
case err := <-errChan:
t.Errorf("Error: %v", err)
case <-time.After(ConcurrentBuildControllersTestWait):
if atomic.LoadInt32(&buildModifiedCount) != 1 {
t.Errorf("The build was modified an unexpected number of times. Got: %d, Expected: 1", buildModifiedCount)
}
if atomic.LoadInt32(&podAddedCount) != 1 {
t.Errorf("The build pod was created an unexpected number of times. Got: %d, Expected: 1", podAddedCount)
}
}
}
func runImageChangeTriggerTest(t *testing.T, clusterAdminClient *client.Client, imageStream *imageapi.ImageStream, imageStreamMapping *imageapi.ImageStreamMapping, config *buildapi.BuildConfig, tag string) {
created, err := clusterAdminClient.BuildConfigs(testutil.Namespace()).Create(config)
if err != nil {
t.Fatalf("Couldn't create BuildConfig: %v", err)
}
watch, err := clusterAdminClient.Builds(testutil.Namespace()).Watch(labels.Everything(), fields.Everything(), created.ResourceVersion)
if err != nil {
t.Fatalf("Couldn't subscribe to Builds %v", err)
}
watch2, err := clusterAdminClient.BuildConfigs(testutil.Namespace()).Watch(labels.Everything(), fields.Everything(), created.ResourceVersion)
if err != nil {
t.Fatalf("Couldn't subscribe to BuildConfigs %v", err)
}
defer watch2.Stop()
imageStream, err = clusterAdminClient.ImageStreams(testutil.Namespace()).Create(imageStream)
if err != nil {
t.Fatalf("Couldn't create ImageStream: %v", err)
}
err = clusterAdminClient.ImageStreamMappings(testutil.Namespace()).Create(imageStreamMapping)
if err != nil {
t.Fatalf("Couldn't create Image: %v", err)
}
// wait for initial build event from the creation of the imagerepo with tag latest
event := waitForWatch(t, "initial build added", watch)
if e, a := watchapi.Added, event.Type; e != a {
t.Fatalf("expected watch event type %s, got %s", e, a)
}
newBuild := event.Object.(*buildapi.Build)
switch newBuild.Spec.Strategy.Type {
case buildapi.SourceBuildStrategyType:
if newBuild.Spec.Strategy.SourceStrategy.From.Name != "registry:8080/openshift/test-image-trigger:"+tag {
i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name)
bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name)
t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\ntrigger is %s\n", "registry:8080/openshift/test-image-trigger:"+tag, newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[0].ImageChange)
}
case buildapi.DockerBuildStrategyType:
if newBuild.Spec.Strategy.DockerStrategy.From.Name != "registry:8080/openshift/test-image-trigger:"+tag {
i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name)
bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name)
t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\ntrigger is %s\n", "registry:8080/openshift/test-image-trigger:"+tag, newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[0].ImageChange)
}
case buildapi.CustomBuildStrategyType:
if newBuild.Spec.Strategy.CustomStrategy.From.Name != "registry:8080/openshift/test-image-trigger:"+tag {
i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name)
bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name)
t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\ntrigger is %s\n", "registry:8080/openshift/test-image-trigger:"+tag, newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[0].ImageChange)
}
}
// Wait for an update on the specific build that was added
watch3, err := clusterAdminClient.Builds(testutil.Namespace()).Watch(labels.Everything(), fields.OneTermEqualSelector("name", newBuild.Name), newBuild.ResourceVersion)
defer watch3.Stop()
if err != nil {
t.Fatalf("Couldn't subscribe to Builds %v", err)
}
event = waitForWatch(t, "initial build update", watch3)
if e, a := watchapi.Modified, event.Type; e != a {
t.Fatalf("expected watch event type %s, got %s", e, a)
}
newBuild = event.Object.(*buildapi.Build)
// Make sure the resolution of the build's docker image pushspec didn't mutate the persisted API object
if newBuild.Spec.Output.To.Name != "test-image-trigger-repo:outputtag" {
t.Fatalf("unexpected build output: %#v %#v", newBuild.Spec.Output.To, newBuild.Spec.Output)
}
if newBuild.Labels["testlabel"] != "testvalue" {
t.Fatalf("Expected build with label %s=%s from build config got %s=%s", "testlabel", "testvalue", "testlabel", newBuild.Labels["testlabel"])
}
// wait for build config to be updated
WaitLoop:
for {
select {
case e := <-watch2.ResultChan():
event = &e
continue
case <-time.After(BuildControllersWatchTimeout):
break WaitLoop
}
}
updatedConfig := event.Object.(*buildapi.BuildConfig)
if err != nil {
t.Fatalf("Couldn't get BuildConfig: %v", err)
}
// the first tag did not have an image id, so the last trigger field is the pull spec
if updatedConfig.Spec.Triggers[0].ImageChange.LastTriggeredImageID != "registry:8080/openshift/test-image-trigger:"+tag {
t.Fatalf("Expected imageID equal to pull spec, got %#v", updatedConfig.Spec.Triggers[0].ImageChange)
}
// clear out the build/buildconfig watches before triggering a new build
WaitLoop2:
for {
select {
case <-watch.ResultChan():
continue
case <-watch2.ResultChan():
continue
case <-time.After(BuildControllersWatchTimeout):
break WaitLoop2
}
}
// trigger a build by posting a new image
if err := clusterAdminClient.ImageStreamMappings(testutil.Namespace()).Create(&imageapi.ImageStreamMapping{
ObjectMeta: kapi.ObjectMeta{
Namespace: testutil.Namespace(),
Name: imageStream.Name,
},
Tag: tag,
Image: imageapi.Image{
ObjectMeta: kapi.ObjectMeta{
Name: "ref-2-random",
},
DockerImageReference: "registry:8080/openshift/test-image-trigger:ref-2-random",
},
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
event = waitForWatch(t, "second build created", watch)
if e, a := watchapi.Added, event.Type; e != a {
t.Fatalf("expected watch event type %s, got %s", e, a)
}
newBuild = event.Object.(*buildapi.Build)
switch newBuild.Spec.Strategy.Type {
case buildapi.SourceBuildStrategyType:
if newBuild.Spec.Strategy.SourceStrategy.From.Name != "registry:8080/openshift/test-image-trigger:ref-2-random" {
i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name)
bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name)
t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\trigger is %s\n", "registry:8080/openshift/test-image-trigger:ref-2-random", newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[3].ImageChange)
}
case buildapi.DockerBuildStrategyType:
if newBuild.Spec.Strategy.DockerStrategy.From.Name != "registry:8080/openshift/test-image-trigger:ref-2-random" {
i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name)
bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name)
t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\trigger is %s\n", "registry:8080/openshift/test-image-trigger:ref-2-random", newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[3].ImageChange)
}
case buildapi.CustomBuildStrategyType:
if newBuild.Spec.Strategy.CustomStrategy.From.Name != "registry:8080/openshift/test-image-trigger:ref-2-random" {
i, _ := clusterAdminClient.ImageStreams(testutil.Namespace()).Get(imageStream.Name)
bc, _ := clusterAdminClient.BuildConfigs(testutil.Namespace()).Get(config.Name)
t.Fatalf("Expected build with base image %s, got %s\n, imagerepo is %v\trigger is %s\n", "registry:8080/openshift/test-image-trigger:ref-2-random", newBuild.Spec.Strategy.DockerStrategy.From.Name, i, bc.Spec.Triggers[3].ImageChange)
}
}
// Listen to events on specific build
watch4, err := clusterAdminClient.Builds(testutil.Namespace()).Watch(labels.Everything(), fields.OneTermEqualSelector("name", newBuild.Name), newBuild.ResourceVersion)
defer watch4.Stop()
event = waitForWatch(t, "update on second build", watch4)
if e, a := watchapi.Modified, event.Type; e != a {
t.Fatalf("expected watch event type %s, got %s", e, a)
}
newBuild = event.Object.(*buildapi.Build)
// Make sure the resolution of the build's docker image pushspec didn't mutate the persisted API object
if newBuild.Spec.Output.To.Name != "test-image-trigger-repo:outputtag" {
t.Fatalf("unexpected build output: %#v %#v", newBuild.Spec.Output.To, newBuild.Spec.Output)
}
if newBuild.Labels["testlabel"] != "testvalue" {
t.Fatalf("Expected build with label %s=%s from build config got %s=%s", "testlabel", "testvalue", "testlabel", newBuild.Labels["testlabel"])
}
WaitLoop3:
for {
select {
case e := <-watch2.ResultChan():
event = &e
continue
case <-time.After(BuildControllersWatchTimeout):
break WaitLoop3
}
}
updatedConfig = event.Object.(*buildapi.BuildConfig)
if e, a := "registry:8080/openshift/test-image-trigger:ref-2-random", updatedConfig.Spec.Triggers[0].ImageChange.LastTriggeredImageID; e != a {
t.Errorf("unexpected trigger id: expected %v, got %v", e, a)
}
}
// TestConcurrentBuildPodControllers tests the lifecycle of a build pod when running multiple controllers.
func TestConcurrentBuildPodControllers(t *testing.T) {
// Start a master with multiple BuildPodControllers
osClient, kClient := setupBuildControllerTest(0, 5, 0, t)
ns := testutil.Namespace()
waitTime := ConcurrentBuildPodControllersTestWait
tests := []buildControllerPodTest{
{
Name: "running state test",
States: []buildControllerPodState{
{
PodPhase: kapi.PodRunning,
BuildPhase: buildapi.BuildPhaseRunning,
},
},
},
{
Name: "build succeeded",
States: []buildControllerPodState{
{
PodPhase: kapi.PodRunning,
BuildPhase: buildapi.BuildPhaseRunning,
},
{
PodPhase: kapi.PodSucceeded,
BuildPhase: buildapi.BuildPhaseComplete,
},
},
},
{
Name: "build failed",
States: []buildControllerPodState{
{
PodPhase: kapi.PodRunning,
BuildPhase: buildapi.BuildPhaseRunning,
},
{
PodPhase: kapi.PodFailed,
BuildPhase: buildapi.BuildPhaseFailed,
},
},
},
}
for _, test := range tests {
// Setup communications channels
podReadyChan := make(chan *kapi.Pod) // Will receive a value when a build pod is ready
errChan := make(chan error) // Will receive a value when an error occurs
stateReached := int32(0)
// Create a build
b, err := osClient.Builds(ns).Create(mockBuild())
checkErr(t, err)
// Watch build pod for transition to pending
podWatch, err := kClient.Pods(ns).Watch(labels.Everything(), fields.OneTermEqualSelector("metadata.name", buildutil.GetBuildPodName(b)), "")
checkErr(t, err)
go func() {
for e := range podWatch.ResultChan() {
pod, ok := e.Object.(*kapi.Pod)
if !ok {
checkErr(t, fmt.Errorf("%s: unexpected object received: %#v\n", test.Name, e.Object))
}
if pod.Status.Phase == kapi.PodPending {
podReadyChan <- pod
break
}
}
}()
var pod *kapi.Pod
select {
case pod = <-podReadyChan:
if pod.Status.Phase != kapi.PodPending {
t.Errorf("Got wrong pod phase: %s", pod.Status.Phase)
podWatch.Stop()
continue
}
case <-time.After(BuildControllersWatchTimeout):
t.Errorf("Timed out waiting for build pod to be ready")
podWatch.Stop()
continue
}
podWatch.Stop()
for _, state := range test.States {
// Update pod state and verify that corresponding build state happens accordingly
pod, err := kClient.Pods(ns).Get(pod.Name)
checkErr(t, err)
pod.Status.Phase = state.PodPhase
_, err = kClient.Pods(ns).UpdateStatus(pod)
checkErr(t, err)
buildWatch, err := osClient.Builds(ns).Watch(labels.Everything(), fields.OneTermEqualSelector("name", b.Name), b.ResourceVersion)
checkErr(t, err)
defer buildWatch.Stop()
go func() {
done := false
for e := range buildWatch.ResultChan() {
var ok bool
b, ok = e.Object.(*buildapi.Build)
if !ok {
errChan <- fmt.Errorf("%s: unexpected object received: %#v", test.Name, e.Object)
}
if e.Type != watchapi.Modified {
errChan <- fmt.Errorf("%s: unexpected event received: %s, object: %#v", e.Type, e.Object)
}
if done {
errChan <- fmt.Errorf("%s: unexpected build state: %#v", test.Name, e.Object)
} else if b.Status.Phase == state.BuildPhase {
done = true
atomic.StoreInt32(&stateReached, 1)
}
}
}()
select {
case err := <-errChan:
buildWatch.Stop()
t.Errorf("Error: %v\n", test.Name, err)
break
case <-time.After(waitTime):
buildWatch.Stop()
if atomic.LoadInt32(&stateReached) != 1 {
t.Errorf("%s: Did not reach desired build state: %s", test.Name, state.BuildPhase)
break
}
}
}
}
}
// rebootNode takes node name on provider through the following steps using c:
// - ensures the node is ready
// - ensures all pods on the node are running and ready
// - reboots the node (by executing rebootCmd over ssh)
// - ensures the node reaches some non-ready state
// - ensures the node becomes ready again
// - ensures all pods on the node become running and ready again
//
// It returns true through result only if all of the steps pass; at the first
// failed step, it will return false through result and not run the rest.
func rebootNode(c *client.Client, provider, name, rebootCmd string, result chan bool) {
// Setup
ns := api.NamespaceDefault
ps := newPodStore(c, ns, labels.Everything(), fields.OneTermEqualSelector(client.PodHost, name))
defer ps.Stop()
// Get the node initially.
Logf("Getting %s", name)
node, err := c.Nodes().Get(name)
if err != nil {
Logf("Couldn't get node %s", name)
result <- false
return
}
// Node sanity check: ensure it is "ready".
if !waitForNodeToBeReady(c, name, nodeReadyInitialTimeout) {
result <- false
return
}
// Get all the pods on the node.
pods := ps.List()
podNames := make([]string, len(pods))
for i, p := range pods {
podNames[i] = p.ObjectMeta.Name
}
Logf("Node %s has %d pods: %v", name, len(podNames), podNames)
// For each pod, we do a sanity check to ensure it's running / healthy
// now, as that's what we'll be checking later.
if !checkPodsRunningReady(c, ns, podNames, podReadyBeforeTimeout) {
result <- false
return
}
// Reboot the node.
if err = issueSSHCommand(node, provider, rebootCmd); err != nil {
Logf("Error while issuing ssh command: %v", err)
result <- false
return
}
// Wait for some kind of "not ready" status.
if !waitForNodeToBeNotReady(c, name, rebootNodeNotReadyTimeout) {
result <- false
return
}
// Wait for some kind of "ready" status.
if !waitForNodeToBeReady(c, name, rebootNodeReadyAgainTimeout) {
result <- false
return
}
// Ensure all of the pods that we found on this node before the reboot are
// running / healthy.
if !checkPodsRunningReady(c, ns, podNames, rebootPodReadyAgainTimeout) {
result <- false
return
}
Logf("Reboot successful on node %s", name)
result <- true
}
// rebootNode takes node name on provider through the following steps using c:
// - ensures the node is ready
// - ensures all pods on the node are running and ready
// - reboots the node (by executing rebootCmd over ssh)
// - ensures the node reaches some non-ready state
// - ensures the node becomes ready again
// - ensures all pods on the node become running and ready again
//
// It returns true through result only if all of the steps pass; at the first
// failed step, it will return false through result and not run the rest.
func rebootNode(c *client.Client, provider, name, rebootCmd string, result chan bool) {
// Setup
ns := api.NamespaceSystem
ps := newPodStore(c, ns, labels.Everything(), fields.OneTermEqualSelector(client.PodHost, name))
defer ps.Stop()
// Get the node initially.
Logf("Getting %s", name)
node, err := c.Nodes().Get(name)
if err != nil {
Logf("Couldn't get node %s", name)
result <- false
return
}
// Node sanity check: ensure it is "ready".
if !waitForNodeToBeReady(c, name, nodeReadyInitialTimeout) {
result <- false
return
}
// Get all the pods on the node that don't have liveness probe set.
// Liveness probe may cause restart of a pod during node reboot, and the pod may not be running.
pods := ps.List()
podNames := []string{}
for _, p := range pods {
probe := false
for _, c := range p.Spec.Containers {
if c.LivenessProbe != nil {
probe = true
break
}
}
if !probe {
podNames = append(podNames, p.ObjectMeta.Name)
}
}
Logf("Node %s has %d pods: %v", name, len(podNames), podNames)
// For each pod, we do a sanity check to ensure it's running / healthy
// now, as that's what we'll be checking later.
if !checkPodsRunningReady(c, ns, podNames, podReadyBeforeTimeout) {
result <- false
return
}
// Reboot the node.
if err = issueSSHCommand(node, provider, rebootCmd); err != nil {
Logf("Error while issuing ssh command: %v", err)
result <- false
return
}
// Wait for some kind of "not ready" status.
if !waitForNodeToBeNotReady(c, name, rebootNodeNotReadyTimeout) {
result <- false
return
}
// Wait for some kind of "ready" status.
if !waitForNodeToBeReady(c, name, rebootNodeReadyAgainTimeout) {
result <- false
return
}
// Ensure all of the pods that we found on this node before the reboot are
// running / healthy.
if !checkPodsRunningReady(c, ns, podNames, rebootPodReadyAgainTimeout) {
result <- false
return
}
Logf("Reboot successful on node %s", name)
result <- true
}
// NewSourceApiserver creates a config source that watches and pulls from the apiserver.
func NewSourceApiserver(client *client.Client, hostname string, updates chan<- interface{}) {
lw := cache.NewListWatchFromClient(client, "pods", api.NamespaceAll, fields.OneTermEqualSelector(getHostFieldLabel(client.APIVersion()), hostname))
newSourceApiserverFromLW(lw, updates)
}
