func TestNodeConditionsObservedSince(t *testing.T) {
now := metav1.Now()
observedTime := metav1.NewTime(now.Time.Add(-1 * time.Minute))
testCases := map[string]struct {
observedAt nodeConditionsObservedAt
period time.Duration
now time.Time
result []v1.NodeConditionType
}{
"in-period": {
observedAt: nodeConditionsObservedAt{
v1.NodeMemoryPressure: observedTime.Time,
},
period: 2 * time.Minute,
now: now.Time,
result: []v1.NodeConditionType{v1.NodeMemoryPressure},
},
"out-of-period": {
observedAt: nodeConditionsObservedAt{
v1.NodeMemoryPressure: observedTime.Time,
},
period: 30 * time.Second,
now: now.Time,
result: []v1.NodeConditionType{},
},
}
for testName, testCase := range testCases {
actual := nodeConditionsObservedSince(testCase.observedAt, testCase.period, testCase.now)
if !nodeConditionList(actual).Equal(nodeConditionList(testCase.result)) {
t.Errorf("Test case: %s, expected: %v, actual: %v", testName, testCase.result, actual)
}
}
}
// TestGracefulStoreCanDeleteIfExistingGracePeriodZero tests recovery from
// race condition where the graceful delete is unable to complete
// in prior operation, but the pod remains with deletion timestamp
// and grace period set to 0.
func TestGracefulStoreCanDeleteIfExistingGracePeriodZero(t *testing.T) {
deletionTimestamp := metav1.NewTime(time.Now())
deletionGracePeriodSeconds := int64(0)
initialGeneration := int64(1)
pod := &api.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Generation: initialGeneration,
DeletionGracePeriodSeconds: &deletionGracePeriodSeconds,
DeletionTimestamp: &deletionTimestamp,
},
Spec: api.PodSpec{NodeName: "machine"},
}
testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), "test")
destroyFunc, registry := NewTestGenericStoreRegistry(t)
registry.EnableGarbageCollection = false
defaultDeleteStrategy := testRESTStrategy{api.Scheme, names.SimpleNameGenerator, true, false, true}
registry.DeleteStrategy = testGracefulStrategy{defaultDeleteStrategy}
defer destroyFunc()
graceful, gracefulPending, err := rest.BeforeDelete(registry.DeleteStrategy, testContext, pod, api.NewDeleteOptions(0))
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if graceful {
t.Fatalf("graceful should be false if object has DeletionTimestamp and DeletionGracePeriodSeconds is 0")
}
if gracefulPending {
t.Fatalf("gracefulPending should be false if object has DeletionTimestamp and DeletionGracePeriodSeconds is 0")
}
}
func (sb *summaryBuilder) containerInfoV2ToStats(
name string,
info *cadvisorapiv2.ContainerInfo) stats.ContainerStats {
cStats := stats.ContainerStats{
StartTime: metav1.NewTime(info.Spec.CreationTime),
Name: name,
}
cstat, found := sb.latestContainerStats(info)
if !found {
return cStats
}
if info.Spec.HasCpu {
cpuStats := stats.CPUStats{
Time: metav1.NewTime(cstat.Timestamp),
}
if cstat.CpuInst != nil {
cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total
}
if cstat.Cpu != nil {
cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total
}
cStats.CPU = &cpuStats
}
if info.Spec.HasMemory {
pageFaults := cstat.Memory.ContainerData.Pgfault
majorPageFaults := cstat.Memory.ContainerData.Pgmajfault
cStats.Memory = &stats.MemoryStats{
Time: metav1.NewTime(cstat.Timestamp),
UsageBytes: &cstat.Memory.Usage,
WorkingSetBytes: &cstat.Memory.WorkingSet,
RSSBytes: &cstat.Memory.RSS,
PageFaults: &pageFaults,
MajorPageFaults: &majorPageFaults,
}
// availableBytes = memory limit (if known) - workingset
if !isMemoryUnlimited(info.Spec.Memory.Limit) {
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
cStats.Memory.AvailableBytes = &availableBytes
}
}
sb.containerInfoV2FsStats(info, &cStats)
cStats.UserDefinedMetrics = sb.containerInfoV2ToUserDefinedMetrics(info)
return cStats
}
func TestThresholdsMetGracePeriod(t *testing.T) {
now := metav1.Now()
hardThreshold := Threshold{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
}
softThreshold := Threshold{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("2Gi"),
},
GracePeriod: 1 * time.Minute,
}
oldTime := metav1.NewTime(now.Time.Add(-2 * time.Minute))
testCases := map[string]struct {
observedAt thresholdsObservedAt
now time.Time
result []Threshold
}{
"empty": {
observedAt: thresholdsObservedAt{},
now: now.Time,
result: []Threshold{},
},
"hard-threshold-met": {
observedAt: thresholdsObservedAt{
hardThreshold: now.Time,
},
now: now.Time,
result: []Threshold{hardThreshold},
},
"soft-threshold-not-met": {
observedAt: thresholdsObservedAt{
softThreshold: now.Time,
},
now: now.Time,
result: []Threshold{},
},
"soft-threshold-met": {
observedAt: thresholdsObservedAt{
softThreshold: oldTime.Time,
},
now: now.Time,
result: []Threshold{softThreshold},
},
}
for testName, testCase := range testCases {
actual := thresholdsMetGracePeriod(testCase.observedAt, now.Time)
if !thresholdList(actual).Equal(thresholdList(testCase.result)) {
t.Errorf("Test case: %s, expected: %v, actual: %v", testName, testCase.result, actual)
}
}
}
// setNodeDiskPressureCondition for the node.
// TODO: this needs to move somewhere centralized...
func (kl *Kubelet) setNodeDiskPressureCondition(node *v1.Node) {
currentTime := metav1.NewTime(kl.clock.Now())
var condition *v1.NodeCondition
// Check if NodeDiskPressure condition already exists and if it does, just pick it up for update.
for i := range node.Status.Conditions {
if node.Status.Conditions[i].Type == v1.NodeDiskPressure {
condition = &node.Status.Conditions[i]
}
}
newCondition := false
// If the NodeDiskPressure condition doesn't exist, create one
if condition == nil {
condition = &v1.NodeCondition{
Type: v1.NodeDiskPressure,
Status: v1.ConditionUnknown,
}
// cannot be appended to node.Status.Conditions here because it gets
// copied to the slice. So if we append to the slice here none of the
// updates we make below are reflected in the slice.
newCondition = true
}
// Update the heartbeat time
condition.LastHeartbeatTime = currentTime
// Note: The conditions below take care of the case when a new NodeDiskPressure condition is
// created and as well as the case when the condition already exists. When a new condition
// is created its status is set to v1.ConditionUnknown which matches either
// condition.Status != v1.ConditionTrue or
// condition.Status != v1.ConditionFalse in the conditions below depending on whether
// the kubelet is under disk pressure or not.
if kl.evictionManager.IsUnderDiskPressure() {
if condition.Status != v1.ConditionTrue {
condition.Status = v1.ConditionTrue
condition.Reason = "KubeletHasDiskPressure"
condition.Message = "kubelet has disk pressure"
condition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasDiskPressure")
}
} else {
if condition.Status != v1.ConditionFalse {
condition.Status = v1.ConditionFalse
condition.Reason = "KubeletHasNoDiskPressure"
condition.Message = "kubelet has no disk pressure"
condition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasNoDiskPressure")
}
}
if newCondition {
node.Status.Conditions = append(node.Status.Conditions, *condition)
}
}
// TestActiveDeadlineHandler verifies the active deadline handler functions as expected.
func TestActiveDeadlineHandler(t *testing.T) {
pods := newTestPods(4)
fakeClock := clock.NewFakeClock(time.Now())
podStatusProvider := &mockPodStatusProvider{pods: pods}
fakeRecorder := &record.FakeRecorder{}
handler, err := newActiveDeadlineHandler(podStatusProvider, fakeRecorder, fakeClock)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
now := metav1.Now()
startTime := metav1.NewTime(now.Time.Add(-1 * time.Minute))
// this pod has exceeded its active deadline
exceededActiveDeadlineSeconds := int64(30)
pods[0].Status.StartTime = &startTime
pods[0].Spec.ActiveDeadlineSeconds = &exceededActiveDeadlineSeconds
// this pod has not exceeded its active deadline
notYetActiveDeadlineSeconds := int64(120)
pods[1].Status.StartTime = &startTime
pods[1].Spec.ActiveDeadlineSeconds = ¬YetActiveDeadlineSeconds
// this pod has no deadline
pods[2].Status.StartTime = &startTime
pods[2].Spec.ActiveDeadlineSeconds = nil
testCases := []struct {
pod *v1.Pod
expected bool
}{{pods[0], true}, {pods[1], false}, {pods[2], false}, {pods[3], false}}
for i, testCase := range testCases {
if actual := handler.ShouldSync(testCase.pod); actual != testCase.expected {
t.Errorf("[%d] ShouldSync expected %#v, got %#v", i, testCase.expected, actual)
}
actual := handler.ShouldEvict(testCase.pod)
if actual.Evict != testCase.expected {
t.Errorf("[%d] ShouldEvict.Evict expected %#v, got %#v", i, testCase.expected, actual.Evict)
}
if testCase.expected {
if actual.Reason != reason {
t.Errorf("[%d] ShouldEvict.Reason expected %#v, got %#v", i, message, actual.Reason)
}
if actual.Message != message {
t.Errorf("[%d] ShouldEvict.Message expected %#v, got %#v", i, message, actual.Message)
}
}
}
}
func TestValidateObjectMetaUpdateIgnoresCreationTimestamp(t *testing.T) {
if errs := ValidateObjectMetaUpdate(
&metav1.ObjectMeta{Name: "test", ResourceVersion: "1"},
&metav1.ObjectMeta{Name: "test", ResourceVersion: "1", CreationTimestamp: metav1.NewTime(time.Unix(10, 0))},
field.NewPath("field"),
); len(errs) != 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if errs := ValidateObjectMetaUpdate(
&metav1.ObjectMeta{Name: "test", ResourceVersion: "1", CreationTimestamp: metav1.NewTime(time.Unix(10, 0))},
&metav1.ObjectMeta{Name: "test", ResourceVersion: "1"},
field.NewPath("field"),
); len(errs) != 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if errs := ValidateObjectMetaUpdate(
&metav1.ObjectMeta{Name: "test", ResourceVersion: "1", CreationTimestamp: metav1.NewTime(time.Unix(10, 0))},
&metav1.ObjectMeta{Name: "test", ResourceVersion: "1", CreationTimestamp: metav1.NewTime(time.Unix(11, 0))},
field.NewPath("field"),
); len(errs) != 0 {
t.Fatalf("unexpected errors: %v", errs)
}
}
// markAsDeleting sets the obj's DeletionGracePeriodSeconds to 0, and sets the
// DeletionTimestamp to "now". Finalizers are watching for such updates and will
// finalize the object if their IDs are present in the object's Finalizers list.
func markAsDeleting(obj runtime.Object) (err error) {
objectMeta, kerr := metav1.ObjectMetaFor(obj)
if kerr != nil {
return kerr
}
now := metav1.NewTime(time.Now())
// This handles Generation bump for resources that don't support graceful deletion. For resources that support graceful deletion is handle in pkg/api/rest/delete.go
if objectMeta.DeletionTimestamp == nil && objectMeta.Generation > 0 {
objectMeta.Generation++
}
objectMeta.DeletionTimestamp = &now
var zero int64 = 0
objectMeta.DeletionGracePeriodSeconds = &zero
return nil
}
func (sb *summaryBuilder) containerInfoV2ToUserDefinedMetrics(info *cadvisorapiv2.ContainerInfo) []stats.UserDefinedMetric {
type specVal struct {
ref stats.UserDefinedMetricDescriptor
valType cadvisorapiv1.DataType
time time.Time
value float64
}
udmMap := map[string]*specVal{}
for _, spec := range info.Spec.CustomMetrics {
udmMap[spec.Name] = &specVal{
ref: stats.UserDefinedMetricDescriptor{
Name: spec.Name,
Type: stats.UserDefinedMetricType(spec.Type),
Units: spec.Units,
},
valType: spec.Format,
}
}
for _, stat := range info.Stats {
for name, values := range stat.CustomMetrics {
specVal, ok := udmMap[name]
if !ok {
glog.Warningf("spec for custom metric %q is missing from cAdvisor output. Spec: %+v, Metrics: %+v", name, info.Spec, stat.CustomMetrics)
continue
}
for _, value := range values {
// Pick the most recent value
if value.Timestamp.Before(specVal.time) {
continue
}
specVal.time = value.Timestamp
specVal.value = value.FloatValue
if specVal.valType == cadvisorapiv1.IntType {
specVal.value = float64(value.IntValue)
}
}
}
}
var udm []stats.UserDefinedMetric
for _, specVal := range udmMap {
udm = append(udm, stats.UserDefinedMetric{
UserDefinedMetricDescriptor: specVal.ref,
Time: metav1.NewTime(specVal.time),
Value: specVal.value,
})
}
return udm
}
func TestThresholdsFirstObservedAt(t *testing.T) {
hardThreshold := Threshold{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
}
now := metav1.Now()
oldTime := metav1.NewTime(now.Time.Add(-1 * time.Minute))
testCases := map[string]struct {
thresholds []Threshold
lastObservedAt thresholdsObservedAt
now time.Time
result thresholdsObservedAt
}{
"empty": {
thresholds: []Threshold{},
lastObservedAt: thresholdsObservedAt{},
now: now.Time,
result: thresholdsObservedAt{},
},
"no-previous-observation": {
thresholds: []Threshold{hardThreshold},
lastObservedAt: thresholdsObservedAt{},
now: now.Time,
result: thresholdsObservedAt{
hardThreshold: now.Time,
},
},
"previous-observation": {
thresholds: []Threshold{hardThreshold},
lastObservedAt: thresholdsObservedAt{
hardThreshold: oldTime.Time,
},
now: now.Time,
result: thresholdsObservedAt{
hardThreshold: oldTime.Time,
},
},
}
for testName, testCase := range testCases {
actual := thresholdsFirstObservedAt(testCase.thresholds, testCase.lastObservedAt, testCase.now)
if !reflect.DeepEqual(actual, testCase.result) {
t.Errorf("Test case: %s, expected: %v, actual: %v", testName, testCase.result, actual)
}
}
}
// SetNodeCondition updates specific node condition with patch operation.
func SetNodeCondition(c clientset.Interface, node types.NodeName, condition v1.NodeCondition) error {
generatePatch := func(condition v1.NodeCondition) ([]byte, error) {
raw, err := json.Marshal(&[]v1.NodeCondition{condition})
if err != nil {
return nil, err
}
return []byte(fmt.Sprintf(`{"status":{"conditions":%s}}`, raw)), nil
}
condition.LastHeartbeatTime = metav1.NewTime(time.Now())
patch, err := generatePatch(condition)
if err != nil {
return nil
}
_, err = c.Core().Nodes().PatchStatus(string(node), patch)
return err
}
func newPod(now time.Time, ready bool, beforeSec int) v1.Pod {
conditionStatus := v1.ConditionFalse
if ready {
conditionStatus = v1.ConditionTrue
}
return v1.Pod{
Status: v1.PodStatus{
Conditions: []v1.PodCondition{
{
Type: v1.PodReady,
LastTransitionTime: metav1.NewTime(now.Add(-1 * time.Duration(beforeSec) * time.Second)),
Status: conditionStatus,
},
},
},
}
}
// buildSummaryPods aggregates and returns the container stats in cinfos by the Pod managing the container.
// Containers not managed by a Pod are omitted.
func (sb *summaryBuilder) buildSummaryPods() []stats.PodStats {
// Map each container to a pod and update the PodStats with container data
podToStats := map[stats.PodReference]*stats.PodStats{}
for key, cinfo := range sb.infos {
// on systemd using devicemapper each mount into the container has an associated cgroup.
// we ignore them to ensure we do not get duplicate entries in our summary.
// for details on .mount units: http://man7.org/linux/man-pages/man5/systemd.mount.5.html
if strings.HasSuffix(key, ".mount") {
continue
}
// Build the Pod key if this container is managed by a Pod
if !sb.isPodManagedContainer(&cinfo) {
continue
}
ref := sb.buildPodRef(&cinfo)
// Lookup the PodStats for the pod using the PodRef. If none exists, initialize a new entry.
podStats, found := podToStats[ref]
if !found {
podStats = &stats.PodStats{PodRef: ref}
podToStats[ref] = podStats
}
// Update the PodStats entry with the stats from the container by adding it to stats.Containers
containerName := types.GetContainerName(cinfo.Spec.Labels)
if containerName == leaky.PodInfraContainerName {
// Special case for infrastructure container which is hidden from the user and has network stats
podStats.Network = sb.containerInfoV2ToNetworkStats("pod:"+ref.Namespace+"_"+ref.Name, &cinfo)
podStats.StartTime = metav1.NewTime(cinfo.Spec.CreationTime)
} else {
podStats.Containers = append(podStats.Containers, sb.containerInfoV2ToStats(containerName, &cinfo))
}
}
// Add each PodStats to the result
result := make([]stats.PodStats, 0, len(podToStats))
for _, podStats := range podToStats {
// Lookup the volume stats for each pod
podUID := kubetypes.UID(podStats.PodRef.UID)
if vstats, found := sb.fsResourceAnalyzer.GetPodVolumeStats(podUID); found {
podStats.VolumeStats = vstats.Volumes
}
result = append(result, *podStats)
}
return result
}
// EventAggregate identifies similar events and groups into a common event if required
func (e *EventAggregator) EventAggregate(newEvent *v1.Event) (*v1.Event, error) {
aggregateKey, localKey := e.keyFunc(newEvent)
now := metav1.NewTime(e.clock.Now())
record := aggregateRecord{localKeys: sets.NewString(), lastTimestamp: now}
e.Lock()
defer e.Unlock()
value, found := e.cache.Get(aggregateKey)
if found {
record = value.(aggregateRecord)
}
// if the last event was far enough in the past, it is not aggregated, and we must reset state
maxInterval := time.Duration(e.maxIntervalInSeconds) * time.Second
interval := now.Time.Sub(record.lastTimestamp.Time)
if interval > maxInterval {
record = aggregateRecord{localKeys: sets.NewString()}
}
record.localKeys.Insert(localKey)
record.lastTimestamp = now
e.cache.Add(aggregateKey, record)
if record.localKeys.Len() < e.maxEvents {
return newEvent, nil
}
// do not grow our local key set any larger than max
record.localKeys.PopAny()
// create a new aggregate event
eventCopy := &v1.Event{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%v.%x", newEvent.InvolvedObject.Name, now.UnixNano()),
Namespace: newEvent.Namespace,
},
Count: 1,
FirstTimestamp: now,
InvolvedObject: newEvent.InvolvedObject,
LastTimestamp: now,
Message: e.messageFunc(newEvent),
Type: newEvent.Type,
Reason: newEvent.Reason,
Source: newEvent.Source,
}
return eventCopy, nil
}
func TestNodeConditionsLastObservedAt(t *testing.T) {
now := metav1.Now()
oldTime := metav1.NewTime(now.Time.Add(-1 * time.Minute))
testCases := map[string]struct {
nodeConditions []v1.NodeConditionType
lastObservedAt nodeConditionsObservedAt
now time.Time
result nodeConditionsObservedAt
}{
"no-previous-observation": {
nodeConditions: []v1.NodeConditionType{v1.NodeMemoryPressure},
lastObservedAt: nodeConditionsObservedAt{},
now: now.Time,
result: nodeConditionsObservedAt{
v1.NodeMemoryPressure: now.Time,
},
},
"previous-observation": {
nodeConditions: []v1.NodeConditionType{v1.NodeMemoryPressure},
lastObservedAt: nodeConditionsObservedAt{
v1.NodeMemoryPressure: oldTime.Time,
},
now: now.Time,
result: nodeConditionsObservedAt{
v1.NodeMemoryPressure: now.Time,
},
},
"old-observation": {
nodeConditions: []v1.NodeConditionType{},
lastObservedAt: nodeConditionsObservedAt{
v1.NodeMemoryPressure: oldTime.Time,
},
now: now.Time,
result: nodeConditionsObservedAt{
v1.NodeMemoryPressure: oldTime.Time,
},
},
}
for testName, testCase := range testCases {
actual := nodeConditionsLastObservedAt(testCase.nodeConditions, testCase.lastObservedAt, testCase.now)
if !reflect.DeepEqual(actual, testCase.result) {
t.Errorf("Test case: %s, expected: %v, actual: %v", testName, testCase.result, actual)
}
}
}
func TestNewPodAddedDelete(t *testing.T) {
channel, ch, _ := createPodConfigTester(PodConfigNotificationIncremental)
// should register an add
addedPod := CreateValidPod("foo", "new")
podUpdate := CreatePodUpdate(kubetypes.ADD, TestSource, addedPod)
channel <- podUpdate
expectPodUpdate(t, ch, CreatePodUpdate(kubetypes.ADD, TestSource, addedPod))
// mark this pod as deleted
timestamp := metav1.NewTime(time.Now())
deletedPod := CreateValidPod("foo", "new")
deletedPod.ObjectMeta.DeletionTimestamp = ×tamp
podUpdate = CreatePodUpdate(kubetypes.DELETE, TestSource, deletedPod)
channel <- podUpdate
// the existing pod should be gracefully deleted
expectPodUpdate(t, ch, CreatePodUpdate(kubetypes.DELETE, TestSource, addedPod))
}
// eventObserve records the event, and determines if its frequency should update
func (e *eventLogger) eventObserve(newEvent *v1.Event) (*v1.Event, []byte, error) {
var (
patch []byte
err error
)
key := getEventKey(newEvent)
eventCopy := *newEvent
event := &eventCopy
e.Lock()
defer e.Unlock()
lastObservation := e.lastEventObservationFromCache(key)
// we have seen this event before, so we must prepare a patch
if lastObservation.count > 0 {
// update the event based on the last observation so patch will work as desired
event.Name = lastObservation.name
event.ResourceVersion = lastObservation.resourceVersion
event.FirstTimestamp = lastObservation.firstTimestamp
event.Count = int32(lastObservation.count) + 1
eventCopy2 := *event
eventCopy2.Count = 0
eventCopy2.LastTimestamp = metav1.NewTime(time.Unix(0, 0))
newData, _ := json.Marshal(event)
oldData, _ := json.Marshal(eventCopy2)
patch, err = strategicpatch.CreateStrategicMergePatch(oldData, newData, event)
}
// record our new observation
e.cache.Add(
key,
eventLog{
count: int(event.Count),
firstTimestamp: event.FirstTimestamp,
name: event.Name,
resourceVersion: event.ResourceVersion,
},
)
return event, patch, err
}
func TestNewStatusPreservesPodStartTime(t *testing.T) {
syncer := newTestManager(&fake.Clientset{})
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
UID: "12345678",
Name: "foo",
Namespace: "new",
},
Status: v1.PodStatus{},
}
now := metav1.Now()
startTime := metav1.NewTime(now.Time.Add(-1 * time.Minute))
pod.Status.StartTime = &startTime
syncer.SetPodStatus(pod, getRandomPodStatus())
status := expectPodStatus(t, syncer, pod)
if !status.StartTime.Time.Equal(startTime.Time) {
t.Errorf("Unexpected start time, expected %v, actual %v", startTime, status.StartTime)
}
}
func TestPodDescribeResultsSorted(t *testing.T) {
// Arrange
fake := fake.NewSimpleClientset(
&api.EventList{
Items: []api.Event{
{
ObjectMeta: metav1.ObjectMeta{Name: "one"},
Source: api.EventSource{Component: "kubelet"},
Message: "Item 1",
FirstTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)),
LastTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
{
ObjectMeta: metav1.ObjectMeta{Name: "two"},
Source: api.EventSource{Component: "scheduler"},
Message: "Item 2",
FirstTimestamp: metav1.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)),
LastTimestamp: metav1.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
{
ObjectMeta: metav1.ObjectMeta{Name: "three"},
Source: api.EventSource{Component: "kubelet"},
Message: "Item 3",
FirstTimestamp: metav1.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)),
LastTimestamp: metav1.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
},
},
&api.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}},
)
c := &describeClient{T: t, Namespace: "foo", Interface: fake}
d := PodDescriber{c}
// Act
out, err := d.Describe("foo", "bar", DescriberSettings{ShowEvents: true})
// Assert
if err != nil {
t.Errorf("unexpected error: %v", err)
}
VerifyDatesInOrder(out, "\n" /* rowDelimiter */, "\t" /* columnDelimiter */, t)
}
func (sb *summaryBuilder) containerInfoV2ToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *stats.NetworkStats {
if !info.Spec.HasNetwork {
return nil
}
cstat, found := sb.latestContainerStats(info)
if !found {
return nil
}
for _, inter := range cstat.Network.Interfaces {
if inter.Name == network.DefaultInterfaceName {
return &stats.NetworkStats{
Time: metav1.NewTime(cstat.Timestamp),
RxBytes: &inter.RxBytes,
RxErrors: &inter.RxErrors,
TxBytes: &inter.TxBytes,
TxErrors: &inter.TxErrors,
}
}
}
glog.V(4).Infof("Missing default interface %q for %s", network.DefaultInterfaceName, name)
return nil
}
func (a *HorizontalController) updateStatus(hpa *autoscaling.HorizontalPodAutoscaler, currentReplicas, desiredReplicas int32, cpuCurrentUtilization *int32, cmStatus string, rescale bool) error {
hpa.Status = autoscaling.HorizontalPodAutoscalerStatus{
CurrentReplicas: currentReplicas,
DesiredReplicas: desiredReplicas,
CurrentCPUUtilizationPercentage: cpuCurrentUtilization,
LastScaleTime: hpa.Status.LastScaleTime,
}
if cmStatus != "" {
hpa.Annotations[HpaCustomMetricsStatusAnnotationName] = cmStatus
}
if rescale {
now := metav1.NewTime(time.Now())
hpa.Status.LastScaleTime = &now
}
_, err := a.hpaNamespacer.HorizontalPodAutoscalers(hpa.Namespace).UpdateStatus(hpa)
if err != nil {
a.eventRecorder.Event(hpa, v1.EventTypeWarning, "FailedUpdateStatus", err.Error())
return fmt.Errorf("failed to update status for %s: %v", hpa.Name, err)
}
glog.V(2).Infof("Successfully updated status for %s", hpa.Name)
return nil
}
func TestSortableEvents(t *testing.T) {
// Arrange
list := SortableEvents([]api.Event{
{
Source: api.EventSource{Component: "kubelet"},
Message: "Item 1",
FirstTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)),
LastTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
{
Source: api.EventSource{Component: "scheduler"},
Message: "Item 2",
FirstTimestamp: metav1.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)),
LastTimestamp: metav1.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
{
Source: api.EventSource{Component: "kubelet"},
Message: "Item 3",
FirstTimestamp: metav1.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)),
LastTimestamp: metav1.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
})
// Act
sort.Sort(list)
// Assert
if list[0].Message != "Item 2" ||
list[1].Message != "Item 3" ||
list[2].Message != "Item 1" {
t.Fatal("List is not sorted by time. List: ", list)
}
}
func TestCustomMetrics(t *testing.T) {
spec := []v1.MetricSpec{
{
Name: "qos",
Type: v1.MetricGauge,
Format: v1.IntType,
Units: "per second",
},
{
Name: "cpuLoad",
Type: v1.MetricCumulative,
Format: v1.FloatType,
Units: "count",
},
}
timestamp1 := time.Now()
timestamp2 := time.Now().Add(time.Minute)
metrics := map[string][]v1.MetricVal{
"qos": {
{
Timestamp: timestamp1,
IntValue: 10,
},
{
Timestamp: timestamp2,
IntValue: 100,
},
},
"cpuLoad": {
{
Timestamp: timestamp1,
FloatValue: 1.2,
},
{
Timestamp: timestamp2,
FloatValue: 2.1,
},
},
}
cInfo := v2.ContainerInfo{
Spec: v2.ContainerSpec{
CustomMetrics: spec,
},
Stats: []*v2.ContainerStats{
{
CustomMetrics: metrics,
},
},
}
sb := &summaryBuilder{}
assert.Contains(t, sb.containerInfoV2ToUserDefinedMetrics(&cInfo),
kubestats.UserDefinedMetric{
UserDefinedMetricDescriptor: kubestats.UserDefinedMetricDescriptor{
Name: "qos",
Type: kubestats.MetricGauge,
Units: "per second",
},
Time: metav1.NewTime(timestamp2),
Value: 100,
},
kubestats.UserDefinedMetric{
UserDefinedMetricDescriptor: kubestats.UserDefinedMetricDescriptor{
Name: "cpuLoad",
Type: kubestats.MetricCumulative,
Units: "count",
},
Time: metav1.NewTime(timestamp2),
Value: 2.1,
})
}
func TestPodLogOptions(t *testing.T) {
sinceSeconds := int64(1)
sinceTime := metav1.NewTime(time.Date(2000, 1, 1, 12, 34, 56, 0, time.UTC).Local())
tailLines := int64(2)
limitBytes := int64(3)
versionedLogOptions := &v1.PodLogOptions{
Container: "mycontainer",
Follow: true,
Previous: true,
SinceSeconds: &sinceSeconds,
SinceTime: &sinceTime,
Timestamps: true,
TailLines: &tailLines,
LimitBytes: &limitBytes,
}
unversionedLogOptions := &api.PodLogOptions{
Container: "mycontainer",
Follow: true,
Previous: true,
SinceSeconds: &sinceSeconds,
SinceTime: &sinceTime,
Timestamps: true,
TailLines: &tailLines,
LimitBytes: &limitBytes,
}
expectedParameters := url.Values{
"container": {"mycontainer"},
"follow": {"true"},
"previous": {"true"},
"sinceSeconds": {"1"},
"sinceTime": {"2000-01-01T12:34:56Z"},
"timestamps": {"true"},
"tailLines": {"2"},
"limitBytes": {"3"},
}
codec := runtime.NewParameterCodec(api.Scheme)
// unversioned -> query params
{
actualParameters, err := codec.EncodeParameters(unversionedLogOptions, v1.SchemeGroupVersion)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actualParameters, expectedParameters) {
t.Fatalf("Expected\n%#v\ngot\n%#v", expectedParameters, actualParameters)
}
}
// versioned -> query params
{
actualParameters, err := codec.EncodeParameters(versionedLogOptions, v1.SchemeGroupVersion)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(actualParameters, expectedParameters) {
t.Fatalf("Expected\n%#v\ngot\n%#v", expectedParameters, actualParameters)
}
}
// query params -> versioned
{
convertedLogOptions := &v1.PodLogOptions{}
err := codec.DecodeParameters(expectedParameters, v1.SchemeGroupVersion, convertedLogOptions)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(convertedLogOptions, versionedLogOptions) {
t.Fatalf("Unexpected deserialization:\n%s", diff.ObjectGoPrintSideBySide(versionedLogOptions, convertedLogOptions))
}
}
// query params -> unversioned
{
convertedLogOptions := &api.PodLogOptions{}
err := codec.DecodeParameters(expectedParameters, v1.SchemeGroupVersion, convertedLogOptions)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(convertedLogOptions, unversionedLogOptions) {
t.Fatalf("Unexpected deserialization:\n%s", diff.ObjectGoPrintSideBySide(unversionedLogOptions, convertedLogOptions))
}
}
}
// Set OODCondition for the node.
func (kl *Kubelet) setNodeOODCondition(node *v1.Node) {
currentTime := metav1.NewTime(kl.clock.Now())
var nodeOODCondition *v1.NodeCondition
// Check if NodeOutOfDisk condition already exists and if it does, just pick it up for update.
for i := range node.Status.Conditions {
if node.Status.Conditions[i].Type == v1.NodeOutOfDisk {
nodeOODCondition = &node.Status.Conditions[i]
}
}
newOODCondition := false
// If the NodeOutOfDisk condition doesn't exist, create one.
if nodeOODCondition == nil {
nodeOODCondition = &v1.NodeCondition{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionUnknown,
}
// nodeOODCondition cannot be appended to node.Status.Conditions here because it gets
// copied to the slice. So if we append nodeOODCondition to the slice here none of the
// updates we make to nodeOODCondition below are reflected in the slice.
newOODCondition = true
}
// Update the heartbeat time irrespective of all the conditions.
nodeOODCondition.LastHeartbeatTime = currentTime
// Note: The conditions below take care of the case when a new NodeOutOfDisk condition is
// created and as well as the case when the condition already exists. When a new condition
// is created its status is set to v1.ConditionUnknown which matches either
// nodeOODCondition.Status != v1.ConditionTrue or
// nodeOODCondition.Status != v1.ConditionFalse in the conditions below depending on whether
// the kubelet is out of disk or not.
if kl.isOutOfDisk() {
if nodeOODCondition.Status != v1.ConditionTrue {
nodeOODCondition.Status = v1.ConditionTrue
nodeOODCondition.Reason = "KubeletOutOfDisk"
nodeOODCondition.Message = "out of disk space"
nodeOODCondition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeOutOfDisk")
}
} else {
if nodeOODCondition.Status != v1.ConditionFalse {
// Update the out of disk condition when the condition status is unknown even if we
// are within the outOfDiskTransitionFrequency duration. We do this to set the
// condition status correctly at kubelet startup.
if nodeOODCondition.Status == v1.ConditionUnknown || kl.clock.Since(nodeOODCondition.LastTransitionTime.Time) >= kl.outOfDiskTransitionFrequency {
nodeOODCondition.Status = v1.ConditionFalse
nodeOODCondition.Reason = "KubeletHasSufficientDisk"
nodeOODCondition.Message = "kubelet has sufficient disk space available"
nodeOODCondition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasSufficientDisk")
} else {
glog.Infof("Node condition status for OutOfDisk is false, but last transition time is less than %s", kl.outOfDiskTransitionFrequency)
}
}
}
if newOODCondition {
node.Status.Conditions = append(node.Status.Conditions, *nodeOODCondition)
}
}
// Set Ready condition for the node.
func (kl *Kubelet) setNodeReadyCondition(node *v1.Node) {
// NOTE(aaronlevy): NodeReady condition needs to be the last in the list of node conditions.
// This is due to an issue with version skewed kubelet and master components.
// ref: https://github.com/kubernetes/kubernetes/issues/16961
currentTime := metav1.NewTime(kl.clock.Now())
var newNodeReadyCondition v1.NodeCondition
rs := append(kl.runtimeState.runtimeErrors(), kl.runtimeState.networkErrors()...)
if len(rs) == 0 {
newNodeReadyCondition = v1.NodeCondition{
Type: v1.NodeReady,
Status: v1.ConditionTrue,
Reason: "KubeletReady",
Message: "kubelet is posting ready status",
LastHeartbeatTime: currentTime,
}
} else {
newNodeReadyCondition = v1.NodeCondition{
Type: v1.NodeReady,
Status: v1.ConditionFalse,
Reason: "KubeletNotReady",
Message: strings.Join(rs, ","),
LastHeartbeatTime: currentTime,
}
}
// Append AppArmor status if it's enabled.
// TODO(timstclair): This is a temporary message until node feature reporting is added.
if newNodeReadyCondition.Status == v1.ConditionTrue &&
kl.appArmorValidator != nil && kl.appArmorValidator.ValidateHost() == nil {
newNodeReadyCondition.Message = fmt.Sprintf("%s. AppArmor enabled", newNodeReadyCondition.Message)
}
// Record any soft requirements that were not met in the container manager.
status := kl.containerManager.Status()
if status.SoftRequirements != nil {
newNodeReadyCondition.Message = fmt.Sprintf("%s. WARNING: %s", newNodeReadyCondition.Message, status.SoftRequirements.Error())
}
readyConditionUpdated := false
needToRecordEvent := false
for i := range node.Status.Conditions {
if node.Status.Conditions[i].Type == v1.NodeReady {
if node.Status.Conditions[i].Status == newNodeReadyCondition.Status {
newNodeReadyCondition.LastTransitionTime = node.Status.Conditions[i].LastTransitionTime
} else {
newNodeReadyCondition.LastTransitionTime = currentTime
needToRecordEvent = true
}
node.Status.Conditions[i] = newNodeReadyCondition
readyConditionUpdated = true
break
}
}
if !readyConditionUpdated {
newNodeReadyCondition.LastTransitionTime = currentTime
node.Status.Conditions = append(node.Status.Conditions, newNodeReadyCondition)
}
if needToRecordEvent {
if newNodeReadyCondition.Status == v1.ConditionTrue {
kl.recordNodeStatusEvent(v1.EventTypeNormal, events.NodeReady)
} else {
kl.recordNodeStatusEvent(v1.EventTypeNormal, events.NodeNotReady)
glog.Infof("Node became not ready: %+v", newNodeReadyCondition)
}
}
}
// initialNode constructs the initial v1.Node for this Kubelet, incorporating node
// labels, information from the cloud provider, and Kubelet configuration.
func (kl *Kubelet) initialNode() (*v1.Node, error) {
node := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: string(kl.nodeName),
Labels: map[string]string{
metav1.LabelHostname: kl.hostname,
metav1.LabelOS: goruntime.GOOS,
metav1.LabelArch: goruntime.GOARCH,
metav1.LabelFluentdDsReady: "true",
},
},
Spec: v1.NodeSpec{
Unschedulable: !kl.registerSchedulable,
},
}
if len(kl.kubeletConfiguration.RegisterWithTaints) > 0 {
annotations := make(map[string]string)
taints := make([]v1.Taint, len(kl.kubeletConfiguration.RegisterWithTaints))
for i := range kl.kubeletConfiguration.RegisterWithTaints {
if err := v1.Convert_api_Taint_To_v1_Taint(&kl.kubeletConfiguration.RegisterWithTaints[i], &taints[i], nil); err != nil {
return nil, err
}
}
b, err := json.Marshal(taints)
if err != nil {
return nil, err
}
annotations[v1.TaintsAnnotationKey] = string(b)
node.ObjectMeta.Annotations = annotations
}
// Initially, set NodeNetworkUnavailable to true.
if kl.providerRequiresNetworkingConfiguration() {
node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{
Type: v1.NodeNetworkUnavailable,
Status: v1.ConditionTrue,
Reason: "NoRouteCreated",
Message: "Node created without a route",
LastTransitionTime: metav1.NewTime(kl.clock.Now()),
})
}
if kl.enableControllerAttachDetach {
if node.Annotations == nil {
node.Annotations = make(map[string]string)
}
glog.Infof("Setting node annotation to enable volume controller attach/detach")
node.Annotations[volumehelper.ControllerManagedAttachAnnotation] = "true"
} else {
glog.Infof("Controller attach/detach is disabled for this node; Kubelet will attach and detach volumes")
}
// @question: should this be place after the call to the cloud provider? which also applies labels
for k, v := range kl.nodeLabels {
if cv, found := node.ObjectMeta.Labels[k]; found {
glog.Warningf("the node label %s=%s will overwrite default setting %s", k, v, cv)
}
node.ObjectMeta.Labels[k] = v
}
if kl.cloud != nil {
instances, ok := kl.cloud.Instances()
if !ok {
return nil, fmt.Errorf("failed to get instances from cloud provider")
}
// TODO(roberthbailey): Can we do this without having credentials to talk
// to the cloud provider?
// TODO: ExternalID is deprecated, we'll have to drop this code
externalID, err := instances.ExternalID(kl.nodeName)
if err != nil {
return nil, fmt.Errorf("failed to get external ID from cloud provider: %v", err)
}
node.Spec.ExternalID = externalID
// TODO: We can't assume that the node has credentials to talk to the
// cloudprovider from arbitrary nodes. At most, we should talk to a
// local metadata server here.
node.Spec.ProviderID, err = cloudprovider.GetInstanceProviderID(kl.cloud, kl.nodeName)
if err != nil {
return nil, err
}
instanceType, err := instances.InstanceType(kl.nodeName)
if err != nil {
return nil, err
}
if instanceType != "" {
glog.Infof("Adding node label from cloud provider: %s=%s", metav1.LabelInstanceType, instanceType)
node.ObjectMeta.Labels[metav1.LabelInstanceType] = instanceType
}
// If the cloud has zone information, label the node with the zone information
zones, ok := kl.cloud.Zones()
if ok {
zone, err := zones.GetZone()
if err != nil {
return nil, fmt.Errorf("failed to get zone from cloud provider: %v", err)
}
if zone.FailureDomain != "" {
glog.Infof("Adding node label from cloud provider: %s=%s", metav1.LabelZoneFailureDomain, zone.FailureDomain)
node.ObjectMeta.Labels[metav1.LabelZoneFailureDomain] = zone.FailureDomain
}
if zone.Region != "" {
glog.Infof("Adding node label from cloud provider: %s=%s", metav1.LabelZoneRegion, zone.Region)
node.ObjectMeta.Labels[metav1.LabelZoneRegion] = zone.Region
}
}
} else {
node.Spec.ExternalID = kl.hostname
if kl.autoDetectCloudProvider {
// If no cloud provider is defined - use the one detected by cadvisor
info, err := kl.GetCachedMachineInfo()
if err == nil {
kl.updateCloudProviderFromMachineInfo(node, info)
}
}
}
kl.setNodeStatus(node)
return node, nil
}
func TestDescribeContainers(t *testing.T) {
testCases := []struct {
container api.Container
status api.ContainerStatus
expectedElements []string
}{
// Running state.
{
container: api.Container{Name: "test", Image: "image"},
status: api.ContainerStatus{
Name: "test",
State: api.ContainerState{
Running: &api.ContainerStateRunning{
StartedAt: metav1.NewTime(time.Now()),
},
},
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Running", "Ready", "True", "Restart Count", "7", "Image", "image", "Started"},
},
// Waiting state.
{
container: api.Container{Name: "test", Image: "image"},
status: api.ContainerStatus{
Name: "test",
State: api.ContainerState{
Waiting: &api.ContainerStateWaiting{
Reason: "potato",
},
},
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "Reason", "potato"},
},
// Terminated state.
{
container: api.Container{Name: "test", Image: "image"},
status: api.ContainerStatus{
Name: "test",
State: api.ContainerState{
Terminated: &api.ContainerStateTerminated{
StartedAt: metav1.NewTime(time.Now()),
FinishedAt: metav1.NewTime(time.Now()),
Reason: "potato",
ExitCode: 2,
},
},
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Terminated", "Ready", "True", "Restart Count", "7", "Image", "image", "Reason", "potato", "Started", "Finished", "Exit Code", "2"},
},
// Last Terminated
{
container: api.Container{Name: "test", Image: "image"},
status: api.ContainerStatus{
Name: "test",
State: api.ContainerState{
Running: &api.ContainerStateRunning{
StartedAt: metav1.NewTime(time.Now()),
},
},
LastTerminationState: api.ContainerState{
Terminated: &api.ContainerStateTerminated{
StartedAt: metav1.NewTime(time.Now().Add(time.Second * 3)),
FinishedAt: metav1.NewTime(time.Now()),
Reason: "crashing",
ExitCode: 3,
},
},
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Terminated", "Ready", "True", "Restart Count", "7", "Image", "image", "Started", "Finished", "Exit Code", "2", "crashing", "3"},
},
// No state defaults to waiting.
{
container: api.Container{Name: "test", Image: "image"},
status: api.ContainerStatus{
Name: "test",
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image"},
},
// Env
{
container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{ConfigMapRef: &api.ConfigMapEnvSource{LocalObjectReference: api.LocalObjectReference{Name: "a123"}}}}},
status: api.ContainerStatus{
Name: "test",
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "envname", "xyz", "a123\tConfigMap"},
},
{
container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{Prefix: "p_", ConfigMapRef: &api.ConfigMapEnvSource{LocalObjectReference: api.LocalObjectReference{Name: "a123"}}}}},
status: api.ContainerStatus{
Name: "test",
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "envname", "xyz", "a123\tConfigMap with prefix 'p_'"},
},
{
container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{SecretRef: &api.SecretEnvSource{LocalObjectReference: api.LocalObjectReference{Name: "a123"}}}}},
status: api.ContainerStatus{
Name: "test",
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "envname", "xyz", "a123\tSecret"},
},
{
container: api.Container{Name: "test", Image: "image", Env: []api.EnvVar{{Name: "envname", Value: "xyz"}}, EnvFrom: []api.EnvFromSource{{Prefix: "p_", SecretRef: &api.SecretEnvSource{LocalObjectReference: api.LocalObjectReference{Name: "a123"}}}}},
status: api.ContainerStatus{
Name: "test",
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "envname", "xyz", "a123\tSecret with prefix 'p_'"},
},
// Command
{
container: api.Container{Name: "test", Image: "image", Command: []string{"sleep", "1000"}},
status: api.ContainerStatus{
Name: "test",
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "sleep", "1000"},
},
// Args
{
container: api.Container{Name: "test", Image: "image", Args: []string{"time", "1000"}},
status: api.ContainerStatus{
Name: "test",
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"test", "State", "Waiting", "Ready", "True", "Restart Count", "7", "Image", "image", "time", "1000"},
},
// Using limits.
{
container: api.Container{
Name: "test",
Image: "image",
Resources: api.ResourceRequirements{
Limits: api.ResourceList{
api.ResourceName(api.ResourceCPU): resource.MustParse("1000"),
api.ResourceName(api.ResourceMemory): resource.MustParse("4G"),
api.ResourceName(api.ResourceStorage): resource.MustParse("20G"),
},
},
},
status: api.ContainerStatus{
Name: "test",
Ready: true,
RestartCount: 7,
},
expectedElements: []string{"cpu", "1k", "memory", "4G", "storage", "20G"},
},
// Using requests.
{
container: api.Container{
Name: "test",
Image: "image",
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceCPU): resource.MustParse("1000"),
api.ResourceName(api.ResourceMemory): resource.MustParse("4G"),
api.ResourceName(api.ResourceStorage): resource.MustParse("20G"),
},
},
},
expectedElements: []string{"cpu", "1k", "memory", "4G", "storage", "20G"},
},
}
for i, testCase := range testCases {
out := new(bytes.Buffer)
pod := api.Pod{
Spec: api.PodSpec{
Containers: []api.Container{testCase.container},
},
Status: api.PodStatus{
ContainerStatuses: []api.ContainerStatus{testCase.status},
},
}
writer := &PrefixWriter{out}
describeContainers("Containers", pod.Spec.Containers, pod.Status.ContainerStatuses, EnvValueRetriever(&pod), writer, "")
output := out.String()
for _, expected := range testCase.expectedElements {
if !strings.Contains(output, expected) {
t.Errorf("Test case %d: expected to find %q in output: %q", i, expected, output)
}
}
}
}
// TestEventCorrelator validates proper counting, aggregation of events
func TestEventCorrelator(t *testing.T) {
firstEvent := makeEvent("first", "i am first", makeObjectReference("Pod", "my-pod", "my-ns"))
duplicateEvent := makeEvent("duplicate", "me again", makeObjectReference("Pod", "my-pod", "my-ns"))
uniqueEvent := makeEvent("unique", "snowflake", makeObjectReference("Pod", "my-pod", "my-ns"))
similarEvent := makeEvent("similar", "similar message", makeObjectReference("Pod", "my-pod", "my-ns"))
aggregateEvent := makeEvent(similarEvent.Reason, EventAggregatorByReasonMessageFunc(&similarEvent), similarEvent.InvolvedObject)
scenario := map[string]struct {
previousEvents []v1.Event
newEvent v1.Event
expectedEvent v1.Event
intervalSeconds int
}{
"create-a-single-event": {
previousEvents: []v1.Event{},
newEvent: firstEvent,
expectedEvent: setCount(firstEvent, 1),
intervalSeconds: 5,
},
"the-same-event-should-just-count": {
previousEvents: makeEvents(1, duplicateEvent),
newEvent: duplicateEvent,
expectedEvent: setCount(duplicateEvent, 2),
intervalSeconds: 5,
},
"the-same-event-should-just-count-even-if-more-than-aggregate": {
previousEvents: makeEvents(defaultAggregateMaxEvents, duplicateEvent),
newEvent: duplicateEvent,
expectedEvent: setCount(duplicateEvent, defaultAggregateMaxEvents+1),
intervalSeconds: 5,
},
"create-many-unique-events": {
previousEvents: makeUniqueEvents(30),
newEvent: uniqueEvent,
expectedEvent: setCount(uniqueEvent, 1),
intervalSeconds: 5,
},
"similar-events-should-aggregate-event": {
previousEvents: makeSimilarEvents(defaultAggregateMaxEvents-1, similarEvent, similarEvent.Message),
newEvent: similarEvent,
expectedEvent: setCount(aggregateEvent, 1),
intervalSeconds: 5,
},
"similar-events-many-times-should-count-the-aggregate": {
previousEvents: makeSimilarEvents(defaultAggregateMaxEvents, similarEvent, similarEvent.Message),
newEvent: similarEvent,
expectedEvent: setCount(aggregateEvent, 2),
intervalSeconds: 5,
},
"similar-events-whose-interval-is-greater-than-aggregate-interval-do-not-aggregate": {
previousEvents: makeSimilarEvents(defaultAggregateMaxEvents-1, similarEvent, similarEvent.Message),
newEvent: similarEvent,
expectedEvent: setCount(similarEvent, 1),
intervalSeconds: defaultAggregateIntervalInSeconds,
},
}
for testScenario, testInput := range scenario {
eventInterval := time.Duration(testInput.intervalSeconds) * time.Second
clock := clock.IntervalClock{Time: time.Now(), Duration: eventInterval}
correlator := NewEventCorrelator(&clock)
for i := range testInput.previousEvents {
event := testInput.previousEvents[i]
now := metav1.NewTime(clock.Now())
event.FirstTimestamp = now
event.LastTimestamp = now
result, err := correlator.EventCorrelate(&event)
if err != nil {
t.Errorf("scenario %v: unexpected error playing back prevEvents %v", testScenario, err)
}
correlator.UpdateState(result.Event)
}
// update the input to current clock value
now := metav1.NewTime(clock.Now())
testInput.newEvent.FirstTimestamp = now
testInput.newEvent.LastTimestamp = now
result, err := correlator.EventCorrelate(&testInput.newEvent)
if err != nil {
t.Errorf("scenario %v: unexpected error correlating input event %v", testScenario, err)
}
_, err = validateEvent(testScenario, result.Event, &testInput.expectedEvent, t)
if err != nil {
t.Errorf("scenario %v: unexpected error validating result %v", testScenario, err)
}
}
}
func TestDescribeEvents(t *testing.T) {
events := &api.EventList{
Items: []api.Event{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "foo",
},
Source: api.EventSource{Component: "kubelet"},
Message: "Item 1",
FirstTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)),
LastTimestamp: metav1.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
},
}
m := map[string]Describer{
"DaemonSetDescriber": &DaemonSetDescriber{
fake.NewSimpleClientset(&extensions.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
Namespace: "foo",
},
}, events),
},
"DeploymentDescriber": &DeploymentDescriber{
fake.NewSimpleClientset(events),
versionedfake.NewSimpleClientset(&v1beta1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
Namespace: "foo",
},
Spec: v1beta1.DeploymentSpec{
Replicas: util.Int32Ptr(1),
Selector: &metav1.LabelSelector{},
},
}),
},
"EndpointsDescriber": &EndpointsDescriber{
fake.NewSimpleClientset(&api.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
Namespace: "foo",
},
}, events),
},
// TODO(jchaloup): add tests for:
// - HorizontalPodAutoscalerDescriber
// - IngressDescriber
// - JobDescriber
"NodeDescriber": &NodeDescriber{
fake.NewSimpleClientset(&api.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
SelfLink: "url/url/url",
},
}, events),
},
"PersistentVolumeDescriber": &PersistentVolumeDescriber{
fake.NewSimpleClientset(&api.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
SelfLink: "url/url/url",
},
}, events),
},
"PodDescriber": &PodDescriber{
fake.NewSimpleClientset(&api.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
Namespace: "foo",
SelfLink: "url/url/url",
},
}, events),
},
"ReplicaSetDescriber": &ReplicaSetDescriber{
fake.NewSimpleClientset(&extensions.ReplicaSet{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
Namespace: "foo",
},
}, events),
},
"ReplicationControllerDescriber": &ReplicationControllerDescriber{
fake.NewSimpleClientset(&api.ReplicationController{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
Namespace: "foo",
},
}, events),
},
"Service": &ServiceDescriber{
fake.NewSimpleClientset(&api.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
Namespace: "foo",
},
}, events),
},
"StorageClass": &StorageClassDescriber{
fake.NewSimpleClientset(&storage.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: "bar",
},
}, events),
},
}
for name, d := range m {
out, err := d.Describe("foo", "bar", DescriberSettings{ShowEvents: true})
if err != nil {
t.Errorf("unexpected error for %q: %v", name, err)
}
if !strings.Contains(out, "bar") {
t.Errorf("unexpected out for %q: %s", name, out)
}
if !strings.Contains(out, "Events:") {
t.Errorf("events not found for %q when ShowEvents=true: %s", name, out)
}
out, err = d.Describe("foo", "bar", DescriberSettings{ShowEvents: false})
if err != nil {
t.Errorf("unexpected error for %q: %s", name, err)
}
if !strings.Contains(out, "bar") {
t.Errorf("unexpected out for %q: %s", name, out)
}
if strings.Contains(out, "Events:") {
t.Errorf("events found for %q when ShowEvents=false: %s", name, out)
}
}
}