// NewFakeControllerExpectationsLookup creates a fake store for PodExpectations.
func NewFakeControllerExpectationsLookup(ttl time.Duration) (*ControllerExpectations, *util.FakeClock) {
fakeTime := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
fakeClock := util.NewFakeClock(fakeTime)
ttlPolicy := &cache.TTLPolicy{Ttl: ttl, Clock: fakeClock}
ttlStore := cache.NewFakeExpirationStore(
ExpKeyFunc, nil, ttlPolicy, fakeClock)
return &ControllerExpectations{ttlStore}, fakeClock
}
func newTestBasicWorkQueue() (*basicWorkQueue, *util.FakeClock) {
fakeClock := util.NewFakeClock(time.Now())
wq := &basicWorkQueue{
clock: fakeClock,
queue: make(map[types.UID]time.Time),
}
return wq, fakeClock
}
// TestSecondsSinceSync verifies that proper results are returned
// when checking the time between syncs
func TestSecondsSinceSync(t *testing.T) {
tunneler := &SSHTunneler{}
assert := assert.New(t)
tunneler.lastSync = time.Date(2015, time.January, 1, 1, 1, 1, 1, time.UTC).Unix()
// Nano Second. No difference.
tunneler.clock = util.NewFakeClock(time.Date(2015, time.January, 1, 1, 1, 1, 2, time.UTC))
assert.Equal(int64(0), tunneler.SecondsSinceSync())
// Second
tunneler.clock = util.NewFakeClock(time.Date(2015, time.January, 1, 1, 1, 2, 1, time.UTC))
assert.Equal(int64(1), tunneler.SecondsSinceSync())
// Minute
tunneler.clock = util.NewFakeClock(time.Date(2015, time.January, 1, 1, 2, 1, 1, time.UTC))
assert.Equal(int64(60), tunneler.SecondsSinceSync())
// Hour
tunneler.clock = util.NewFakeClock(time.Date(2015, time.January, 1, 2, 1, 1, 1, time.UTC))
assert.Equal(int64(3600), tunneler.SecondsSinceSync())
// Day
tunneler.clock = util.NewFakeClock(time.Date(2015, time.January, 2, 1, 1, 1, 1, time.UTC))
assert.Equal(int64(86400), tunneler.SecondsSinceSync())
// Month
tunneler.clock = util.NewFakeClock(time.Date(2015, time.February, 1, 1, 1, 1, 1, time.UTC))
assert.Equal(int64(2678400), tunneler.SecondsSinceSync())
// Future Month. Should be -Month.
tunneler.lastSync = time.Date(2015, time.February, 1, 1, 1, 1, 1, time.UTC).Unix()
tunneler.clock = util.NewFakeClock(time.Date(2015, time.January, 1, 1, 1, 1, 1, time.UTC))
assert.Equal(int64(-2678400), tunneler.SecondsSinceSync())
}
func newTestGenericPLEG() *TestGenericPLEG {
fakeRuntime := &containertest.FakeRuntime{}
clock := util.NewFakeClock(time.Time{})
// The channel capacity should be large enough to hold all events in a
// single test.
pleg := &GenericPLEG{
relistPeriod: time.Hour,
runtime: fakeRuntime,
eventChannel: make(chan *PodLifecycleEvent, 100),
podRecords: make(podRecords),
clock: clock,
}
return &TestGenericPLEG{pleg: pleg, runtime: fakeRuntime, clock: clock}
}
func TestAddTwoFireEarly(t *testing.T) {
fakeClock := util.NewFakeClock(time.Now())
q := newDelayingQueue(fakeClock)
first := "foo"
second := "bar"
third := "baz"
q.AddAfter(first, 1*time.Second)
q.AddAfter(second, 50*time.Millisecond)
if err := waitForWaitingQueueToFill(q); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if q.Len() != 0 {
t.Errorf("should not have added")
}
fakeClock.Step(60 * time.Millisecond)
if err := waitForAdded(t, q, 1); err != nil {
t.Fatalf("unexpected err: %v", err)
}
item, _ := q.Get()
if !reflect.DeepEqual(item, second) {
t.Errorf("expected %v, got %v", second, item)
}
q.AddAfter(third, 2*time.Second)
fakeClock.Step(1 * time.Second)
if err := waitForAdded(t, q, 1); err != nil {
t.Fatalf("unexpected err: %v", err)
}
item, _ = q.Get()
if !reflect.DeepEqual(item, first) {
t.Errorf("expected %v, got %v", first, item)
}
fakeClock.Step(2 * time.Second)
if err := waitForAdded(t, q, 1); err != nil {
t.Fatalf("unexpected err: %v", err)
}
item, _ = q.Get()
if !reflect.DeepEqual(item, third) {
t.Errorf("expected %v, got %v", third, item)
}
}
func TestRateLimitingQueue(t *testing.T) {
limiter := NewItemExponentialFailureRateLimiter(1*time.Millisecond, 1*time.Second)
queue := NewRateLimitingQueue(limiter).(*rateLimitingType)
fakeClock := util.NewFakeClock(time.Now())
delayingQueue := &delayingType{
Interface: New(),
clock: fakeClock,
heartbeat: fakeClock.Tick(maxWait),
stopCh: make(chan struct{}),
waitingForAddCh: make(chan waitFor, 1000),
}
queue.DelayingInterface = delayingQueue
queue.AddRateLimited("one")
waitEntry := <-delayingQueue.waitingForAddCh
if e, a := 1*time.Millisecond, waitEntry.readyAt.Sub(fakeClock.Now()); e != a {
t.Errorf("expected %v, got %v", e, a)
}
queue.AddRateLimited("one")
waitEntry = <-delayingQueue.waitingForAddCh
if e, a := 10*time.Millisecond, waitEntry.readyAt.Sub(fakeClock.Now()); e != a {
t.Errorf("expected %v, got %v", e, a)
}
if e, a := 2, queue.NumRequeues("one"); e != a {
t.Errorf("expected %v, got %v", e, a)
}
queue.AddRateLimited("two")
waitEntry = <-delayingQueue.waitingForAddCh
if e, a := 1*time.Millisecond, waitEntry.readyAt.Sub(fakeClock.Now()); e != a {
t.Errorf("expected %v, got %v", e, a)
}
queue.AddRateLimited("two")
waitEntry = <-delayingQueue.waitingForAddCh
if e, a := 10*time.Millisecond, waitEntry.readyAt.Sub(fakeClock.Now()); e != a {
t.Errorf("expected %v, got %v", e, a)
}
queue.Forget("one")
if e, a := 0, queue.NumRequeues("one"); e != a {
t.Errorf("expected %v, got %v", e, a)
}
queue.AddRateLimited("one")
waitEntry = <-delayingQueue.waitingForAddCh
if e, a := 1*time.Millisecond, waitEntry.readyAt.Sub(fakeClock.Now()); e != a {
t.Errorf("expected %v, got %v", e, a)
}
}
func TestGarbageCollectImageNotOldEnough(t *testing.T) {
policy := ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,
MinAge: time.Minute * 1,
}
fakeRuntime := &containertest.FakeRuntime{}
mockCadvisor := new(cadvisortest.Mock)
manager := &realImageManager{
runtime: fakeRuntime,
policy: policy,
imageRecords: make(map[string]*imageRecord),
cadvisor: mockCadvisor,
recorder: &record.FakeRecorder{},
}
fakeRuntime.ImageList = []container.Image{
makeImage(0, 1024),
makeImage(1, 2048),
}
// 1 image is in use, and another one is not old enough
fakeRuntime.AllPodList = []*containertest.FakePod{
{Pod: &container.Pod{
Containers: []*container.Container{
makeContainer(1),
},
}},
}
fakeClock := util.NewFakeClock(time.Now())
t.Log(fakeClock.Now())
require.NoError(t, manager.detectImages(fakeClock.Now()))
require.Equal(t, manager.imageRecordsLen(), 2)
// no space freed since one image is in used, and another one is not old enough
spaceFreed, err := manager.freeSpace(1024, fakeClock.Now())
assert := assert.New(t)
require.NoError(t, err)
assert.EqualValues(0, spaceFreed)
assert.Len(fakeRuntime.ImageList, 2)
// move clock by minAge duration, then 1 image will be garbage collected
fakeClock.Step(policy.MinAge)
spaceFreed, err = manager.freeSpace(1024, fakeClock.Now())
require.NoError(t, err)
assert.EqualValues(1024, spaceFreed)
assert.Len(fakeRuntime.ImageList, 1)
}
func TestSimpleQueue(t *testing.T) {
fakeClock := util.NewFakeClock(time.Now())
q := newDelayingQueue(fakeClock)
first := "foo"
q.AddAfter(first, 50*time.Millisecond)
if err := waitForWaitingQueueToFill(q); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if q.Len() != 0 {
t.Errorf("should not have added")
}
fakeClock.Step(60 * time.Millisecond)
if err := waitForAdded(t, q, 1); err != nil {
t.Errorf("should have added")
}
item, _ := q.Get()
q.Done(item)
// step past the next heartbeat
fakeClock.Step(10 * time.Second)
err := wait.Poll(1*time.Millisecond, 30*time.Millisecond, func() (done bool, err error) {
if q.Len() > 0 {
return false, fmt.Errorf("added to queue")
}
return false, nil
})
if err != wait.ErrWaitTimeout {
t.Errorf("expected timeout, got: %v", err)
}
if q.Len() != 0 {
t.Errorf("should not have added")
}
}
func TestCopyShifting(t *testing.T) {
fakeClock := util.NewFakeClock(time.Now())
q := newDelayingQueue(fakeClock)
first := "foo"
second := "bar"
third := "baz"
q.AddAfter(first, 1*time.Second)
q.AddAfter(second, 500*time.Millisecond)
q.AddAfter(third, 250*time.Millisecond)
if err := waitForWaitingQueueToFill(q); err != nil {
t.Fatalf("unexpected err: %v", err)
}
if q.Len() != 0 {
t.Errorf("should not have added")
}
fakeClock.Step(2 * time.Second)
if err := waitForAdded(t, q, 3); err != nil {
t.Fatalf("unexpected err: %v", err)
}
actualFirst, _ := q.Get()
if !reflect.DeepEqual(actualFirst, third) {
t.Errorf("expected %v, got %v", third, actualFirst)
}
actualSecond, _ := q.Get()
if !reflect.DeepEqual(actualSecond, second) {
t.Errorf("expected %v, got %v", second, actualSecond)
}
actualThird, _ := q.Get()
if !reflect.DeepEqual(actualThird, first) {
t.Errorf("expected %v, got %v", first, actualThird)
}
}
func TestMultiSinkCache(t *testing.T) {
testPod := &api.Pod{
ObjectMeta: api.ObjectMeta{
SelfLink: "/api/version/pods/foo",
Name: "foo",
Namespace: "baz",
UID: "bar",
},
}
testPod2 := &api.Pod{
ObjectMeta: api.ObjectMeta{
SelfLink: "/api/version/pods/foo",
Name: "foo",
Namespace: "baz",
UID: "differentUid",
},
}
testRef, err := api.GetPartialReference(testPod, "spec.containers[2]")
testRef2, err := api.GetPartialReference(testPod2, "spec.containers[3]")
if err != nil {
t.Fatal(err)
}
table := []struct {
obj k8sruntime.Object
eventtype string
reason string
messageFmt string
elements []interface{}
expect *api.Event
expectLog string
expectUpdate bool
}{
{
obj: testRef,
eventtype: api.EventTypeNormal,
reason: "Started",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "bar",
APIVersion: "version",
FieldPath: "spec.containers[2]",
},
Reason: "Started",
Message: "some verbose message: 1",
Source: api.EventSource{Component: "eventTest"},
Count: 1,
Type: api.EventTypeNormal,
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"version", ResourceVersion:"", FieldPath:"spec.containers[2]"}): type: 'Normal' reason: 'Started' some verbose message: 1`,
expectUpdate: false,
},
{
obj: testPod,
eventtype: api.EventTypeNormal,
reason: "Killed",
messageFmt: "some other verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "bar",
APIVersion: "version",
},
Reason: "Killed",
Message: "some other verbose message: 1",
Source: api.EventSource{Component: "eventTest"},
Count: 1,
Type: api.EventTypeNormal,
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"version", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'Killed' some other verbose message: 1`,
expectUpdate: false,
},
{
obj: testRef,
eventtype: api.EventTypeNormal,
reason: "Started",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "bar",
APIVersion: "version",
FieldPath: "spec.containers[2]",
},
Reason: "Started",
Message: "some verbose message: 1",
Source: api.EventSource{Component: "eventTest"},
Count: 2,
Type: api.EventTypeNormal,
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"version", ResourceVersion:"", FieldPath:"spec.containers[2]"}): type: 'Normal' reason: 'Started' some verbose message: 1`,
expectUpdate: true,
},
{
obj: testRef2,
eventtype: api.EventTypeNormal,
reason: "Started",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "differentUid",
APIVersion: "version",
FieldPath: "spec.containers[3]",
},
Reason: "Started",
Message: "some verbose message: 1",
Source: api.EventSource{Component: "eventTest"},
Count: 1,
Type: api.EventTypeNormal,
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"differentUid", APIVersion:"version", ResourceVersion:"", FieldPath:"spec.containers[3]"}): type: 'Normal' reason: 'Started' some verbose message: 1`,
expectUpdate: false,
},
{
obj: testRef,
eventtype: api.EventTypeNormal,
reason: "Started",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "bar",
APIVersion: "version",
FieldPath: "spec.containers[2]",
},
Reason: "Started",
Message: "some verbose message: 1",
Source: api.EventSource{Component: "eventTest"},
Count: 3,
Type: api.EventTypeNormal,
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"bar", APIVersion:"version", ResourceVersion:"", FieldPath:"spec.containers[2]"}): type: 'Normal' reason: 'Started' some verbose message: 1`,
expectUpdate: true,
},
{
obj: testRef2,
eventtype: api.EventTypeNormal,
reason: "Stopped",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "differentUid",
APIVersion: "version",
FieldPath: "spec.containers[3]",
},
Reason: "Stopped",
Message: "some verbose message: 1",
Source: api.EventSource{Component: "eventTest"},
Count: 1,
Type: api.EventTypeNormal,
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"differentUid", APIVersion:"version", ResourceVersion:"", FieldPath:"spec.containers[3]"}): type: 'Normal' reason: 'Stopped' some verbose message: 1`,
expectUpdate: false,
},
{
obj: testRef2,
eventtype: api.EventTypeNormal,
reason: "Stopped",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "baz",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "baz",
UID: "differentUid",
APIVersion: "version",
FieldPath: "spec.containers[3]",
},
Reason: "Stopped",
Message: "some verbose message: 1",
Source: api.EventSource{Component: "eventTest"},
Count: 2,
Type: api.EventTypeNormal,
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"baz", Name:"foo", UID:"differentUid", APIVersion:"version", ResourceVersion:"", FieldPath:"spec.containers[3]"}): type: 'Normal' reason: 'Stopped' some verbose message: 1`,
expectUpdate: true,
},
}
testCache := map[string]*api.Event{}
createEvent := make(chan *api.Event)
updateEvent := make(chan *api.Event)
patchEvent := make(chan *api.Event)
testEvents := testEventSink{
OnCreate: OnCreateFactory(testCache, createEvent),
OnUpdate: func(event *api.Event) (*api.Event, error) {
updateEvent <- event
return event, nil
},
OnPatch: OnPatchFactory(testCache, patchEvent),
}
testCache2 := map[string]*api.Event{}
createEvent2 := make(chan *api.Event)
updateEvent2 := make(chan *api.Event)
patchEvent2 := make(chan *api.Event)
testEvents2 := testEventSink{
OnCreate: OnCreateFactory(testCache2, createEvent2),
OnUpdate: func(event *api.Event) (*api.Event, error) {
updateEvent2 <- event
return event, nil
},
OnPatch: OnPatchFactory(testCache2, patchEvent2),
}
eventBroadcaster := NewBroadcasterForTests(0)
clock := util.NewFakeClock(time.Now())
recorder := recorderWithFakeClock(api.EventSource{Component: "eventTest"}, eventBroadcaster, clock)
sinkWatcher := eventBroadcaster.StartRecordingToSink(&testEvents)
for index, item := range table {
clock.Step(1 * time.Second)
recorder.Eventf(item.obj, item.eventtype, item.reason, item.messageFmt, item.elements...)
// validate event
if item.expectUpdate {
actualEvent := <-patchEvent
validateEvent(string(index), actualEvent, item.expect, t)
} else {
actualEvent := <-createEvent
validateEvent(string(index), actualEvent, item.expect, t)
}
}
// Another StartRecordingToSink call should start to record events with new clean cache.
sinkWatcher2 := eventBroadcaster.StartRecordingToSink(&testEvents2)
for index, item := range table {
clock.Step(1 * time.Second)
recorder.Eventf(item.obj, item.eventtype, item.reason, item.messageFmt, item.elements...)
// validate event
if item.expectUpdate {
actualEvent := <-patchEvent2
validateEvent(string(index), actualEvent, item.expect, t)
} else {
actualEvent := <-createEvent2
validateEvent(string(index), actualEvent, item.expect, t)
}
}
sinkWatcher.Stop()
sinkWatcher2.Stop()
}
func TestEventfNoNamespace(t *testing.T) {
testPod := &api.Pod{
ObjectMeta: api.ObjectMeta{
SelfLink: "/api/version/pods/foo",
Name: "foo",
UID: "bar",
},
}
testRef, err := api.GetPartialReference(testPod, "spec.containers[2]")
if err != nil {
t.Fatal(err)
}
table := []struct {
obj k8sruntime.Object
eventtype string
reason string
messageFmt string
elements []interface{}
expect *api.Event
expectLog string
expectUpdate bool
}{
{
obj: testRef,
eventtype: api.EventTypeNormal,
reason: "Started",
messageFmt: "some verbose message: %v",
elements: []interface{}{1},
expect: &api.Event{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: "default",
},
InvolvedObject: api.ObjectReference{
Kind: "Pod",
Name: "foo",
Namespace: "",
UID: "bar",
APIVersion: "version",
FieldPath: "spec.containers[2]",
},
Reason: "Started",
Message: "some verbose message: 1",
Source: api.EventSource{Component: "eventTest"},
Count: 1,
Type: api.EventTypeNormal,
},
expectLog: `Event(api.ObjectReference{Kind:"Pod", Namespace:"", Name:"foo", UID:"bar", APIVersion:"version", ResourceVersion:"", FieldPath:"spec.containers[2]"}): type: 'Normal' reason: 'Started' some verbose message: 1`,
expectUpdate: false,
},
}
testCache := map[string]*api.Event{}
logCalled := make(chan struct{})
createEvent := make(chan *api.Event)
updateEvent := make(chan *api.Event)
patchEvent := make(chan *api.Event)
testEvents := testEventSink{
OnCreate: OnCreateFactory(testCache, createEvent),
OnUpdate: func(event *api.Event) (*api.Event, error) {
updateEvent <- event
return event, nil
},
OnPatch: OnPatchFactory(testCache, patchEvent),
}
eventBroadcaster := NewBroadcasterForTests(0)
sinkWatcher := eventBroadcaster.StartRecordingToSink(&testEvents)
clock := util.NewFakeClock(time.Now())
recorder := recorderWithFakeClock(api.EventSource{Component: "eventTest"}, eventBroadcaster, clock)
for index, item := range table {
clock.Step(1 * time.Second)
// TODO: uncomment this after we upgrade to Go 1.6.1.
// testing.(*common).log() is racing with testing.(*T).report() in Go 1.6.
// See #23533 for more details.
// logWatcher1 := eventBroadcaster.StartLogging(t.Logf) // Prove that it is useful
logWatcher2 := eventBroadcaster.StartLogging(func(formatter string, args ...interface{}) {
if e, a := item.expectLog, fmt.Sprintf(formatter, args...); e != a {
t.Errorf("Expected '%v', got '%v'", e, a)
}
logCalled <- struct{}{}
})
recorder.Eventf(item.obj, item.eventtype, item.reason, item.messageFmt, item.elements...)
<-logCalled
// validate event
if item.expectUpdate {
actualEvent := <-patchEvent
validateEvent(string(index), actualEvent, item.expect, t)
} else {
actualEvent := <-createEvent
validateEvent(string(index), actualEvent, item.expect, t)
}
// TODO: uncomment this after we upgrade to Go 1.6.1.
// logWatcher1.Stop()
logWatcher2.Stop()
}
sinkWatcher.Stop()
}
// newTestWatchCache just adds a fake clock.
func newTestWatchCache(capacity int) *watchCache {
wc := newWatchCache(capacity)
wc.clock = util.NewFakeClock(time.Now())
return wc
}
func TestSerializedPuller(t *testing.T) {
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: "test_pod",
Namespace: "test-ns",
UID: "bar",
ResourceVersion: "42",
SelfLink: "/api/v1/pods/foo",
}}
cases := []struct {
containerImage string
policy api.PullPolicy
calledFunctions []string
inspectErr error
pullerErr error
expectedErr []error
}{
{ // pull missing image
containerImage: "missing_image",
policy: api.PullIfNotPresent,
calledFunctions: []string{"IsImagePresent", "PullImage"},
inspectErr: nil,
pullerErr: nil,
expectedErr: []error{nil}},
{ // image present, dont pull
containerImage: "present_image",
policy: api.PullIfNotPresent,
calledFunctions: []string{"IsImagePresent"},
inspectErr: nil,
pullerErr: nil,
expectedErr: []error{nil, nil, nil}},
// image present, pull it
{containerImage: "present_image",
policy: api.PullAlways,
calledFunctions: []string{"IsImagePresent", "PullImage"},
inspectErr: nil,
pullerErr: nil,
expectedErr: []error{nil, nil, nil}},
// missing image, error PullNever
{containerImage: "missing_image",
policy: api.PullNever,
calledFunctions: []string{"IsImagePresent"},
inspectErr: nil,
pullerErr: nil,
expectedErr: []error{ErrImageNeverPull, ErrImageNeverPull, ErrImageNeverPull}},
// missing image, unable to inspect
{containerImage: "missing_image",
policy: api.PullIfNotPresent,
calledFunctions: []string{"IsImagePresent"},
inspectErr: errors.New("unknown inspectError"),
pullerErr: nil,
expectedErr: []error{ErrImageInspect, ErrImageInspect, ErrImageInspect}},
// missing image, unable to fetch
{containerImage: "typo_image",
policy: api.PullIfNotPresent,
calledFunctions: []string{"IsImagePresent", "PullImage"},
inspectErr: nil,
pullerErr: errors.New("404"),
expectedErr: []error{ErrImagePull, ErrImagePull, ErrImagePullBackOff, ErrImagePull, ErrImagePullBackOff, ErrImagePullBackOff}},
}
for i, c := range cases {
container := &api.Container{
Name: "container_name",
Image: c.containerImage,
ImagePullPolicy: c.policy,
}
backOff := flowcontrol.NewBackOff(time.Second, time.Minute)
fakeClock := util.NewFakeClock(time.Now())
backOff.Clock = fakeClock
fakeRuntime := &ctest.FakeRuntime{}
fakeRecorder := &record.FakeRecorder{}
puller := NewSerializedImagePuller(fakeRecorder, fakeRuntime, backOff)
fakeRuntime.ImageList = []Image{{"present_image", nil, nil, 0}}
fakeRuntime.Err = c.pullerErr
fakeRuntime.InspectErr = c.inspectErr
for tick, expected := range c.expectedErr {
fakeClock.Step(time.Second)
err, _ := puller.PullImage(pod, container, nil)
fakeRuntime.AssertCalls(c.calledFunctions)
assert.Equal(t, expected, err, "in test %d tick=%d", i, tick)
}
}
}
// TestMemoryPressure
func TestMemoryPressure(t *testing.T) {
podMaker := func(name string, requests api.ResourceList, limits api.ResourceList, memoryWorkingSet string) (*api.Pod, statsapi.PodStats) {
pod := newPod(name, []api.Container{
newContainer(name, requests, api.ResourceList{}),
})
podStats := newPodMemoryStats(pod, resource.MustParse(memoryWorkingSet))
return pod, podStats
}
summaryStatsMaker := func(nodeAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary {
val := resource.MustParse(nodeAvailableBytes)
availableBytes := uint64(val.Value())
result := &statsapi.Summary{
Node: statsapi.NodeStats{
Memory: &statsapi.MemoryStats{
AvailableBytes: &availableBytes,
},
},
Pods: []statsapi.PodStats{},
}
for _, podStat := range podStats {
result.Pods = append(result.Pods, podStat)
}
return result
}
podsToMake := []struct {
name string
requests api.ResourceList
limits api.ResourceList
memoryWorkingSet string
}{
{name: "best-effort-high", requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "500Mi"},
{name: "best-effort-low", requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "300Mi"},
{name: "burstable-high", requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "800Mi"},
{name: "burstable-low", requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "300Mi"},
{name: "guaranteed-high", requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "800Mi"},
{name: "guaranteed-low", requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "200Mi"},
}
pods := []*api.Pod{}
podStats := map[*api.Pod]statsapi.PodStats{}
for _, podToMake := range podsToMake {
pod, podStat := podMaker(podToMake.name, podToMake.requests, podToMake.limits, podToMake.memoryWorkingSet)
pods = append(pods, pod)
podStats[pod] = podStat
}
activePodsFunc := func() []*api.Pod {
return pods
}
fakeClock := util.NewFakeClock(time.Now())
podKiller := &mockPodKiller{}
nodeRef := &api.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
config := Config{
MaxPodGracePeriodSeconds: 5,
PressureTransitionPeriod: time.Minute * 5,
Thresholds: []Threshold{
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("1Gi"),
},
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("2Gi"),
GracePeriod: time.Minute * 2,
},
},
}
summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("2Gi", podStats)}
manager := &managerImpl{
clock: fakeClock,
killPodFunc: podKiller.killPodNow,
config: config,
recorder: &record.FakeRecorder{},
summaryProvider: summaryProvider,
nodeRef: nodeRef,
nodeConditionsLastObservedAt: nodeConditionsObservedAt{},
thresholdsFirstObservedAt: thresholdsObservedAt{},
}
// create a best effort pod to test admission
bestEffortPodToAdmit, _ := podMaker("best-admit", newResourceList("", ""), newResourceList("", ""), "0Gi")
burstablePodToAdmit, _ := podMaker("burst-admit", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi")
// synchronize
manager.synchronize(activePodsFunc)
// we should not have memory pressure
if manager.IsUnderMemoryPressure() {
t.Errorf("Manager should not report memory pressure")
}
// try to admit our pods (they should succeed)
expected := []bool{true, true}
for i, pod := range []*api.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
}
}
// induce soft threshold
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
manager.synchronize(activePodsFunc)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
t.Errorf("Manager should report memory pressure since soft threshold was met")
}
// verify no pod was yet killed because there has not yet been enough time passed.
if podKiller.pod != nil {
t.Errorf("Manager should not have killed a pod yet, but killed: %v", podKiller.pod)
}
// step forward in time pass the grace period
fakeClock.Step(3 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
manager.synchronize(activePodsFunc)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
t.Errorf("Manager should report memory pressure since soft threshold was met")
}
// verify the right pod was killed with the right grace period.
if podKiller.pod != pods[0] {
t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod, pods[0])
}
if podKiller.gracePeriodOverride == nil {
t.Errorf("Manager chose to kill pod but should have had a grace period override.")
}
observedGracePeriod := *podKiller.gracePeriodOverride
if observedGracePeriod != manager.config.MaxPodGracePeriodSeconds {
t.Errorf("Manager chose to kill pod with incorrect grace period. Expected: %d, actual: %d", manager.config.MaxPodGracePeriodSeconds, observedGracePeriod)
}
// reset state
podKiller.pod = nil
podKiller.gracePeriodOverride = nil
// remove memory pressure
fakeClock.Step(20 * time.Minute)
summaryProvider.result = summaryStatsMaker("3Gi", podStats)
manager.synchronize(activePodsFunc)
// we should not have memory pressure
if manager.IsUnderMemoryPressure() {
t.Errorf("Manager should not report memory pressure")
}
// induce memory pressure!
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("500Mi", podStats)
manager.synchronize(activePodsFunc)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
t.Errorf("Manager should report memory pressure")
}
// check the right pod was killed
if podKiller.pod != pods[0] {
t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod, pods[0])
}
observedGracePeriod = *podKiller.gracePeriodOverride
if observedGracePeriod != int64(0) {
t.Errorf("Manager chose to kill pod with incorrect grace period. Expected: %d, actual: %d", 0, observedGracePeriod)
}
// the best-effort pod should not admit, burstable should
expected = []bool{false, true}
for i, pod := range []*api.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
}
}
// reduce memory pressure
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(activePodsFunc)
// we should have memory pressure (because transition period not yet met)
if !manager.IsUnderMemoryPressure() {
t.Errorf("Manager should report memory pressure")
}
// no pod should have been killed
if podKiller.pod != nil {
t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod)
}
// the best-effort pod should not admit, burstable should
expected = []bool{false, true}
for i, pod := range []*api.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
}
}
// move the clock past transition period to ensure that we stop reporting pressure
fakeClock.Step(5 * time.Minute)
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(activePodsFunc)
// we should not have memory pressure (because transition period met)
if manager.IsUnderMemoryPressure() {
t.Errorf("Manager should not report memory pressure")
}
// no pod should have been killed
if podKiller.pod != nil {
t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod)
}
// all pods should admit now
expected = []bool{true, true}
for i, pod := range []*api.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
}
}
}