func TestTryOrdering(t *testing.T) { evictor := NewRateLimitedTimedQueue(util.NewFakeRateLimiter()) evictor.Add("first") evictor.Add("second") evictor.Add("third") order := []string{} count := 0 queued := false evictor.Try(func(value TimedValue) (bool, time.Duration) { count++ if value.AddedAt.IsZero() { t.Fatalf("added should not be zero") } if value.ProcessAt.IsZero() { t.Fatalf("next should not be zero") } if !queued && value.Value == "second" { queued = true return false, time.Millisecond } order = append(order, value.Value) return true, 0 }) if !reflect.DeepEqual(order, []string{"first", "third"}) { t.Fatalf("order was wrong: %v", order) } if count != 3 { t.Fatalf("unexpected iterations: %d", count) } }
func TestTryRemovingWhileTry(t *testing.T) { evictor := NewRateLimitedTimedQueue(util.NewFakeRateLimiter()) evictor.Add("first") evictor.Add("second") evictor.Add("third") processing := make(chan struct{}) wait := make(chan struct{}) order := []string{} count := 0 queued := false // while the Try function is processing "second", remove it from the queue // we should not see "second" retried. go func() { <-processing evictor.Remove("second") close(wait) }() evictor.Try(func(value TimedValue) (bool, time.Duration) { count++ if value.AddedAt.IsZero() { t.Fatalf("added should not be zero") } if value.ProcessAt.IsZero() { t.Fatalf("next should not be zero") } if !queued && value.Value == "second" { queued = true close(processing) <-wait return false, time.Millisecond } order = append(order, value.Value) return true, 0 }) if !reflect.DeepEqual(order, []string{"first", "third"}) { t.Fatalf("order was wrong: %v", order) } if count != 3 { t.Fatalf("unexpected iterations: %d", count) } }
func TestTry(t *testing.T) { evictor := NewRateLimitedTimedQueue(util.NewFakeRateLimiter()) evictor.Add("first") evictor.Add("second") evictor.Add("third") evictor.Remove("second") deletedMap := sets.NewString() evictor.Try(func(value TimedValue) (bool, time.Duration) { deletedMap.Insert(value.Value) return true, 0 }) setPattern := sets.NewString("first", "third") if len(deletedMap) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } if !CheckSetEq(setPattern, deletedMap) { t.Errorf("Invalid map. Got %v, expected %v", deletedMap, setPattern) } }
func TestAddNode(t *testing.T) { evictor := NewRateLimitedTimedQueue(util.NewFakeRateLimiter()) evictor.Add("first") evictor.Add("second") evictor.Add("third") queuePattern := []string{"first", "second", "third"} if len(evictor.queue.queue) != len(queuePattern) { t.Fatalf("Queue %v should have length %d", evictor.queue.queue, len(queuePattern)) } if !CheckQueueEq(queuePattern, evictor.queue.queue) { t.Errorf("Invalid queue. Got %v, expected %v", evictor.queue.queue, queuePattern) } setPattern := sets.NewString("first", "second", "third") if len(evictor.queue.set) != len(setPattern) { t.Fatalf("Map %v should have length %d", evictor.queue.set, len(setPattern)) } if !CheckSetEq(setPattern, evictor.queue.set) { t.Errorf("Invalid map. Got %v, expected %v", evictor.queue.set, setPattern) } }
func startComponents(firstManifestURL, secondManifestURL string) (string, string) { // Setup servers := []string{} glog.Infof("Creating etcd client pointing to %v", servers) handler := delegateHandler{} apiServer := httptest.NewServer(&handler) etcdClient := etcd.NewClient(servers) sleep := 4 * time.Second ok := false for i := 0; i < 3; i++ { keys, err := etcdClient.Get("/", false, false) if err != nil { glog.Warningf("Unable to list root etcd keys: %v", err) if i < 2 { time.Sleep(sleep) sleep = sleep * sleep } continue } for _, node := range keys.Node.Nodes { if _, err := etcdClient.Delete(node.Key, true); err != nil { glog.Fatalf("Unable delete key: %v", err) } } ok = true break } if !ok { glog.Fatalf("Failed to connect to etcd") } cl := client.NewOrDie(&client.Config{Host: apiServer.URL, Version: testapi.Default.GroupAndVersion()}) // TODO: caesarxuchao: hacky way to specify version of Experimental client. // We will fix this by supporting multiple group versions in Config cl.ExtensionsClient = client.NewExtensionsOrDie(&client.Config{Host: apiServer.URL, Version: testapi.Extensions.GroupAndVersion()}) storageVersions := make(map[string]string) etcdStorage, err := master.NewEtcdStorage(etcdClient, latest.GroupOrDie("").InterfacesFor, testapi.Default.GroupAndVersion(), etcdtest.PathPrefix()) storageVersions[""] = testapi.Default.GroupAndVersion() if err != nil { glog.Fatalf("Unable to get etcd storage: %v", err) } expEtcdStorage, err := master.NewEtcdStorage(etcdClient, latest.GroupOrDie("extensions").InterfacesFor, testapi.Extensions.GroupAndVersion(), etcdtest.PathPrefix()) storageVersions["extensions"] = testapi.Extensions.GroupAndVersion() if err != nil { glog.Fatalf("Unable to get etcd storage for experimental: %v", err) } storageDestinations := master.NewStorageDestinations() storageDestinations.AddAPIGroup("", etcdStorage) storageDestinations.AddAPIGroup("extensions", expEtcdStorage) // Master host, port, err := net.SplitHostPort(strings.TrimLeft(apiServer.URL, "http://")) if err != nil { glog.Fatalf("Unable to parse URL '%v': %v", apiServer.URL, err) } portNumber, err := strconv.Atoi(port) if err != nil { glog.Fatalf("Nonnumeric port? %v", err) } publicAddress := net.ParseIP(host) if publicAddress == nil { glog.Fatalf("no public address for %s", host) } // Create a master and install handlers into mux. m := master.New(&master.Config{ StorageDestinations: storageDestinations, KubeletClient: fakeKubeletClient{}, EnableCoreControllers: true, EnableLogsSupport: false, EnableProfiling: true, APIPrefix: "/api", APIGroupPrefix: "/apis", Authorizer: apiserver.NewAlwaysAllowAuthorizer(), AdmissionControl: admit.NewAlwaysAdmit(), ReadWritePort: portNumber, PublicAddress: publicAddress, CacheTimeout: 2 * time.Second, StorageVersions: storageVersions, }) handler.delegate = m.Handler // Scheduler schedulerConfigFactory := factory.NewConfigFactory(cl, nil) schedulerConfig, err := schedulerConfigFactory.Create() if err != nil { glog.Fatalf("Couldn't create scheduler config: %v", err) } eventBroadcaster := record.NewBroadcaster() schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"}) eventBroadcaster.StartLogging(glog.Infof) eventBroadcaster.StartRecordingToSink(cl.Events("")) scheduler.New(schedulerConfig).Run() // ensure the service endpoints are sync'd several times within the window that the integration tests wait go endpointcontroller.NewEndpointController(cl, controller.NoResyncPeriodFunc). Run(3, util.NeverStop) // TODO: Write an integration test for the replication controllers watch. go replicationControllerPkg.NewReplicationManager(cl, controller.NoResyncPeriodFunc, replicationControllerPkg.BurstReplicas). Run(3, util.NeverStop) nodeController := nodecontroller.NewNodeController(nil, cl, 5*time.Minute, util.NewFakeRateLimiter(), util.NewFakeRateLimiter(), 40*time.Second, 60*time.Second, 5*time.Second, nil, false) nodeController.Run(5 * time.Second) cadvisorInterface := new(cadvisor.Fake) // Kubelet (localhost) testRootDir := makeTempDirOrDie("kubelet_integ_1.", "") configFilePath := makeTempDirOrDie("config", testRootDir) glog.Infof("Using %s as root dir for kubelet #1", testRootDir) fakeDocker1.VersionInfo = docker.Env{"ApiVersion=1.20"} kcfg := kubeletapp.SimpleKubelet( cl, &fakeDocker1, "localhost", testRootDir, firstManifestURL, "127.0.0.1", 10250, /* KubeletPort */ 0, /* ReadOnlyPort */ api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, configFilePath, nil, kubecontainer.FakeOS{}, 1*time.Second, /* FileCheckFrequency */ 1*time.Second, /* HTTPCheckFrequency */ 10*time.Second, /* MinimumGCAge */ 3*time.Second, /* NodeStatusUpdateFrequency */ 10*time.Second /* SyncFrequency */) kubeletapp.RunKubelet(kcfg, nil) // Kubelet (machine) // Create a second kubelet so that the guestbook example's two redis slaves both // have a place they can schedule. testRootDir = makeTempDirOrDie("kubelet_integ_2.", "") glog.Infof("Using %s as root dir for kubelet #2", testRootDir) fakeDocker2.VersionInfo = docker.Env{"ApiVersion=1.20"} kcfg = kubeletapp.SimpleKubelet( cl, &fakeDocker2, "127.0.0.1", testRootDir, secondManifestURL, "127.0.0.1", 10251, /* KubeletPort */ 0, /* ReadOnlyPort */ api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, "", nil, kubecontainer.FakeOS{}, 1*time.Second, /* FileCheckFrequency */ 1*time.Second, /* HTTPCheckFrequency */ 10*time.Second, /* MinimumGCAge */ 3*time.Second, /* NodeStatusUpdateFrequency */ 10*time.Second /* SyncFrequency */) kubeletapp.RunKubelet(kcfg, nil) return apiServer.URL, configFilePath }
func TestNodeDeletion(t *testing.T) { fakeNow := unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC) fakeNodeHandler := &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionTrue, // Node status has just been updated. LastHeartbeatTime: fakeNow, LastTransitionTime: fakeNow, }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, Spec: api.NodeSpec{ ExternalID: "node0", }, }, { ObjectMeta: api.ObjectMeta{ Name: "node1", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionTrue, // Node status has just been updated. LastHeartbeatTime: fakeNow, LastTransitionTime: fakeNow, }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, Spec: api.NodeSpec{ ExternalID: "node0", }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0"), *newPod("pod1", "node1")}}), } nodeController := NewNodeController(nil, fakeNodeHandler, 5*time.Minute, util.NewFakeRateLimiter(), util.NewFakeRateLimiter(), testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, false) nodeController.now = func() unversioned.Time { return fakeNow } if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } fakeNodeHandler.Delete("node1") if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } nodeController.podEvictor.Try(func(value TimedValue) (bool, time.Duration) { nodeController.deletePods(value.Value) return true, 0 }) podEvicted := false for _, action := range fakeNodeHandler.Actions() { if action.GetVerb() == "delete" && action.GetResource() == "pods" { podEvicted = true } } if !podEvicted { t.Error("expected pods to be evicted from the deleted node") } }
func TestMonitorNodeStatusUpdateStatus(t *testing.T) { fakeNow := unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC) table := []struct { fakeNodeHandler *FakeNodeHandler timeToPass time.Duration newNodeStatus api.NodeStatus expectedEvictPods bool expectedRequestCount int expectedNodes []*api.Node }{ // Node created long time ago, without status: // Expect Unknown status posted from node controller. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, expectedRequestCount: 2, // List+Update expectedNodes: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, Reason: "NodeStatusNeverUpdated", Message: fmt.Sprintf("Kubelet never posted node status."), LastHeartbeatTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastTransitionTime: fakeNow, }, }, }, }, }, }, // Node created recently, without status. // Expect no action from node controller (within startup grace period). { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: fakeNow, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, expectedRequestCount: 1, // List expectedNodes: nil, }, // Node created long time ago, with status updated by kubelet exceeds grace period. // Expect Unknown status posted from node controller. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionTrue, // Node status hasn't been updated for 1hr. LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, Spec: api.NodeSpec{ ExternalID: "node0", }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, expectedRequestCount: 3, // (List+)List+Update timeToPass: time.Hour, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionTrue, // Node status hasn't been updated for 1hr. LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, expectedNodes: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, Reason: "NodeStatusStopUpdated", Message: fmt.Sprintf("Kubelet stopped posting node status."), LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Time{Time: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC).Add(time.Hour)}, }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, Spec: api.NodeSpec{ ExternalID: "node0", }, }, }, }, // Node created long time ago, with status updated recently. // Expect no action from node controller (within monitor grace period). { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionTrue, // Node status has just been updated. LastHeartbeatTime: fakeNow, LastTransitionTime: fakeNow, }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, Spec: api.NodeSpec{ ExternalID: "node0", }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, expectedRequestCount: 1, // List expectedNodes: nil, }, } for _, item := range table { nodeController := NewNodeController(nil, item.fakeNodeHandler, 5*time.Minute, util.NewFakeRateLimiter(), util.NewFakeRateLimiter(), testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, false) nodeController.now = func() unversioned.Time { return fakeNow } if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } if item.timeToPass > 0 { nodeController.now = func() unversioned.Time { return unversioned.Time{Time: fakeNow.Add(item.timeToPass)} } item.fakeNodeHandler.Existing[0].Status = item.newNodeStatus if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } } if item.expectedRequestCount != item.fakeNodeHandler.RequestCount { t.Errorf("expected %v call, but got %v.", item.expectedRequestCount, item.fakeNodeHandler.RequestCount) } if len(item.fakeNodeHandler.UpdatedNodes) > 0 && !api.Semantic.DeepEqual(item.expectedNodes, item.fakeNodeHandler.UpdatedNodes) { t.Errorf("expected nodes %+v, got %+v", item.expectedNodes[0], item.fakeNodeHandler.UpdatedNodes[0]) } } }
func TestMonitorNodeStatusEvictPods(t *testing.T) { fakeNow := unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC) evictionTimeout := 10 * time.Minute table := []struct { fakeNodeHandler *FakeNodeHandler timeToPass time.Duration newNodeStatus api.NodeStatus expectedEvictPods bool description string }{ // Node created recently, with no status (happens only at cluster startup). { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: fakeNow, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: 0, newNodeStatus: api.NodeStatus{}, expectedEvictPods: false, description: "Node created recently, with no status.", }, // Node created long time ago, and kubelet posted NotReady for a short period of time. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionFalse, LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: evictionTimeout, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionFalse, // Node status has just been updated, and is NotReady for 10min. LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 9, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, expectedEvictPods: false, description: "Node created long time ago, and kubelet posted NotReady for a short period of time.", }, // Node created long time ago, and kubelet posted NotReady for a long period of time. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionFalse, LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: time.Hour, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionFalse, // Node status has just been updated, and is NotReady for 1hr. LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 59, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, expectedEvictPods: true, description: "Node created long time ago, and kubelet posted NotReady for a long period of time.", }, // Node created long time ago, node controller posted Unknown for a short period of time. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: evictionTimeout - testNodeMonitorGracePeriod, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, // Node status was updated by nodecontroller 10min ago LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, expectedEvictPods: false, description: "Node created long time ago, node controller posted Unknown for a short period of time.", }, // Node created long time ago, node controller posted Unknown for a long period of time. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: 60 * time.Minute, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, // Node status was updated by nodecontroller 1hr ago LastHeartbeatTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, expectedEvictPods: true, description: "Node created long time ago, node controller posted Unknown for a long period of time.", }, } for _, item := range table { nodeController := NewNodeController(nil, item.fakeNodeHandler, evictionTimeout, util.NewFakeRateLimiter(), util.NewFakeRateLimiter(), testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, false) nodeController.now = func() unversioned.Time { return fakeNow } if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } if item.timeToPass > 0 { nodeController.now = func() unversioned.Time { return unversioned.Time{Time: fakeNow.Add(item.timeToPass)} } item.fakeNodeHandler.Existing[0].Status = item.newNodeStatus } if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } nodeController.podEvictor.Try(func(value TimedValue) (bool, time.Duration) { remaining, _ := nodeController.deletePods(value.Value) if remaining { nodeController.terminationEvictor.Add(value.Value) } return true, 0 }) nodeController.podEvictor.Try(func(value TimedValue) (bool, time.Duration) { nodeController.terminatePods(value.Value, value.AddedAt) return true, 0 }) podEvicted := false for _, action := range item.fakeNodeHandler.Actions() { if action.GetVerb() == "delete" && action.GetResource() == "pods" { podEvicted = true } } if item.expectedEvictPods != podEvicted { t.Errorf("expected pod eviction: %+v, got %+v for %+v", item.expectedEvictPods, podEvicted, item.description) } } }
// of time. This sample size is not actually large enough to // reliably measure tails (it may give false positives, but not // false negatives), but it should catch low hanging fruit. // // Note that these are fixed and do not depend on the // size of the cluster. Setting parallelTrials larger // distorts the measurements. Perhaps this wouldn't be // true on HA clusters. totalTrials = 200 parallelTrials = 15 minSampleSize = 100 ) // Turn off rate limiting--it interferes with our measurements. oldThrottle := f.Client.RESTClient.Throttle f.Client.RESTClient.Throttle = util.NewFakeRateLimiter() defer func() { f.Client.RESTClient.Throttle = oldThrottle }() failing := sets.NewString() d, err := runServiceLatencies(f, parallelTrials, totalTrials) if err != nil { failing.Insert(fmt.Sprintf("Not all RC/pod/service trials succeeded: %v", err)) } dSorted := durations(d) sort.Sort(dSorted) n := len(dSorted) if n < minSampleSize { failing.Insert(fmt.Sprintf("Did not get a good sample size: %v", dSorted)) } if n < 2 { failing.Insert("Less than two runs succeeded; aborting.")