func TestMonitorNodeStatusUpdateStatus(t *testing.T) { fakeNow := util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC) table := []struct { fakeNodeHandler *FakeNodeHandler timeToPass time.Duration newNodeStatus api.NodeStatus expectedEvictPods bool expectedRequestCount int expectedNodes []*api.Node }{ // Node created long time ago, without status: // Expect Unknown status posted from node controller. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, expectedRequestCount: 2, // List+Update expectedNodes: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, Reason: fmt.Sprintf("Qinglet never posted node status."), LastHeartbeatTime: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastTransitionTime: fakeNow, }, }, }, }, }, }, // Node created recently, without status. // Expect no action from node controller (within startup grace period). { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: fakeNow, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, expectedRequestCount: 1, // List expectedNodes: nil, }, // Node created long time ago, with status updated by qinglet exceeds grace period. // Expect Unknown status posted from node controller. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionTrue, // Node status hasn't been updated for 1hr. LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, Spec: api.NodeSpec{ ExternalID: "node0", }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, expectedRequestCount: 3, // (List+)List+Update timeToPass: time.Hour, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionTrue, // Node status hasn't been updated for 1hr. LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, expectedNodes: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, Reason: fmt.Sprintf("Qinglet stopped posting node status."), LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Time{util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC).Add(time.Hour)}, }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, Spec: api.NodeSpec{ ExternalID: "node0", }, }, }, }, // Node created long time ago, with status updated recently. // Expect no action from node controller (within monitor grace period). { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionTrue, // Node status has just been updated. LastHeartbeatTime: fakeNow, LastTransitionTime: fakeNow, }, }, Capacity: api.ResourceList{ api.ResourceName(api.ResourceCPU): resource.MustParse("10"), api.ResourceName(api.ResourceMemory): resource.MustParse("10G"), }, }, Spec: api.NodeSpec{ ExternalID: "node0", }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, expectedRequestCount: 1, // List expectedNodes: nil, }, } for _, item := range table { nodeController := NewNodeController(nil, item.fakeNodeHandler, 10, 5*time.Minute, NewPodEvictor(util.NewFakeRateLimiter()), testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, false) nodeController.now = func() util.Time { return fakeNow } if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } if item.timeToPass > 0 { nodeController.now = func() util.Time { return util.Time{Time: fakeNow.Add(item.timeToPass)} } item.fakeNodeHandler.Existing[0].Status = item.newNodeStatus if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } } if item.expectedRequestCount != item.fakeNodeHandler.RequestCount { t.Errorf("expected %v call, but got %v.", item.expectedRequestCount, item.fakeNodeHandler.RequestCount) } if len(item.fakeNodeHandler.UpdatedNodes) > 0 && !api.Semantic.DeepEqual(item.expectedNodes, item.fakeNodeHandler.UpdatedNodes) { t.Errorf("expected nodes %+v, got %+v", item.expectedNodes[0], item.fakeNodeHandler.UpdatedNodes[0]) } } }
func TestMonitorNodeStatusEvictPods(t *testing.T) { fakeNow := util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC) evictionTimeout := 10 * time.Minute table := []struct { fakeNodeHandler *FakeNodeHandler timeToPass time.Duration newNodeStatus api.NodeStatus expectedEvictPods bool description string }{ // Node created recently, with no status (happens only at cluster startup). { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: fakeNow, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: 0, newNodeStatus: api.NodeStatus{}, expectedEvictPods: false, description: "Node created recently, with no status.", }, // Node created long time ago, and qinglet posted NotReady for a short period of time. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionFalse, LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: evictionTimeout, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionFalse, // Node status has just been updated, and is NotReady for 10min. LastHeartbeatTime: util.Date(2015, 1, 1, 12, 9, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, expectedEvictPods: false, description: "Node created long time ago, and qinglet posted NotReady for a short period of time.", }, // Node created long time ago, and qinglet posted NotReady for a long period of time. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionFalse, LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: time.Hour, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionFalse, // Node status has just been updated, and is NotReady for 1hr. LastHeartbeatTime: util.Date(2015, 1, 1, 12, 59, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, expectedEvictPods: true, description: "Node created long time ago, and qinglet posted NotReady for a long period of time.", }, // Node created long time ago, node controller posted Unknown for a short period of time. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: evictionTimeout - testNodeMonitorGracePeriod, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, // Node status was updated by nodecontroller 10min ago LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, expectedEvictPods: false, description: "Node created long time ago, node controller posted Unknown for a short period of time.", }, // Node created long time ago, node controller posted Unknown for a long period of time. { fakeNodeHandler: &FakeNodeHandler{ Existing: []*api.Node{ { ObjectMeta: api.ObjectMeta{ Name: "node0", CreationTimestamp: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Fake: testclient.NewSimpleFake(&api.PodList{Items: []api.Pod{*newPod("pod0", "node0")}}), }, timeToPass: 60 * time.Minute, newNodeStatus: api.NodeStatus{ Conditions: []api.NodeCondition{ { Type: api.NodeReady, Status: api.ConditionUnknown, // Node status was updated by nodecontroller 1hr ago LastHeartbeatTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: util.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, expectedEvictPods: true, description: "Node created long time ago, node controller posted Unknown for a long period of time.", }, } for _, item := range table { podEvictor := NewPodEvictor(util.NewFakeRateLimiter()) nodeController := NewNodeController(nil, item.fakeNodeHandler, 10, evictionTimeout, podEvictor, testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, false) nodeController.now = func() util.Time { return fakeNow } if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } if item.timeToPass > 0 { nodeController.now = func() util.Time { return util.Time{Time: fakeNow.Add(item.timeToPass)} } item.fakeNodeHandler.Existing[0].Status = item.newNodeStatus } if err := nodeController.monitorNodeStatus(); err != nil { t.Errorf("unexpected error: %v", err) } podEvictor.TryEvict(func(nodeName string) { nodeController.deletePods(nodeName) }) podEvicted := false for _, action := range item.fakeNodeHandler.Actions { if action.Action == "delete-pod" { podEvicted = true } } if item.expectedEvictPods != podEvicted { t.Errorf("expected pod eviction: %+v, got %+v for %+v", item.expectedEvictPods, podEvicted, item.description) } } }