// BeforeDelete tests whether the object can be gracefully deleted. If graceful is set the object
// should be gracefully deleted, if gracefulPending is set the object has already been gracefully deleted
// (and the provided grace period is longer than the time to deletion), and an error is returned if the
// condition cannot be checked or the gracePeriodSeconds is invalid. The options argument may be updated with
// default values if graceful is true.
func BeforeDelete(strategy RESTDeleteStrategy, ctx api.Context, obj runtime.Object, options *api.DeleteOptions) (graceful, gracefulPending bool, err error) {
objectMeta, gvk, kerr := objectMetaAndKind(strategy, obj)
if kerr != nil {
return false, false, kerr
}
// Checking the Preconditions here to fail early. They'll be enforced later on when we actually do the deletion, too.
if options.Preconditions != nil && options.Preconditions.UID != nil && *options.Preconditions.UID != objectMeta.UID {
return false, false, errors.NewConflict(unversioned.GroupResource{Group: gvk.Group, Resource: gvk.Kind}, objectMeta.Name, fmt.Errorf("the UID in the precondition (%s) does not match the UID in record (%s). The object might have been deleted and then recreated", *options.Preconditions.UID, objectMeta.UID))
}
gracefulStrategy, ok := strategy.(RESTGracefulDeleteStrategy)
if !ok {
return false, false, nil
}
// if the object is already being deleted
if objectMeta.DeletionTimestamp != nil {
// if we are already being deleted, we may only shorten the deletion grace period
// this means the object was gracefully deleted previously but deletionGracePeriodSeconds was not set,
// so we force deletion immediately
if objectMeta.DeletionGracePeriodSeconds == nil {
return false, false, nil
}
// only a shorter grace period may be provided by a user
if options.GracePeriodSeconds != nil {
period := int64(*options.GracePeriodSeconds)
if period > *objectMeta.DeletionGracePeriodSeconds {
return false, true, nil
}
now := unversioned.NewTime(unversioned.Now().Add(time.Second * time.Duration(*options.GracePeriodSeconds)))
objectMeta.DeletionTimestamp = &now
objectMeta.DeletionGracePeriodSeconds = &period
options.GracePeriodSeconds = &period
return true, false, nil
}
// graceful deletion is pending, do nothing
options.GracePeriodSeconds = objectMeta.DeletionGracePeriodSeconds
return false, true, nil
}
if !gracefulStrategy.CheckGracefulDelete(obj, options) {
return false, false, nil
}
now := unversioned.NewTime(unversioned.Now().Add(time.Second * time.Duration(*options.GracePeriodSeconds)))
objectMeta.DeletionTimestamp = &now
objectMeta.DeletionGracePeriodSeconds = options.GracePeriodSeconds
return true, false, nil
}
func (sb *summaryBuilder) containerInfoV2ToStats(
name string,
info *cadvisorapiv2.ContainerInfo) stats.ContainerStats {
cStats := stats.ContainerStats{
StartTime: unversioned.NewTime(info.Spec.CreationTime),
Name: name,
}
cstat, found := sb.latestContainerStats(info)
if !found {
return cStats
}
if info.Spec.HasCpu {
cpuStats := stats.CPUStats{
Time: unversioned.NewTime(cstat.Timestamp),
}
if cstat.CpuInst != nil {
cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total
}
if cstat.Cpu != nil {
cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total
}
cStats.CPU = &cpuStats
}
if info.Spec.HasMemory {
pageFaults := cstat.Memory.ContainerData.Pgfault
majorPageFaults := cstat.Memory.ContainerData.Pgmajfault
cStats.Memory = &stats.MemoryStats{
Time: unversioned.NewTime(cstat.Timestamp),
UsageBytes: &cstat.Memory.Usage,
WorkingSetBytes: &cstat.Memory.WorkingSet,
RSSBytes: &cstat.Memory.RSS,
PageFaults: &pageFaults,
MajorPageFaults: &majorPageFaults,
}
// availableBytes = memory limit (if known) - workingset
if !isMemoryUnlimited(info.Spec.Memory.Limit) {
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
cStats.Memory.AvailableBytes = &availableBytes
}
}
sb.containerInfoV2FsStats(info, &cStats)
cStats.UserDefinedMetrics = sb.containerInfoV2ToUserDefinedMetrics(info)
return cStats
}
func (sb *summaryBuilder) containerInfoV2ToUserDefinedMetrics(info *cadvisorapiv2.ContainerInfo) []stats.UserDefinedMetric {
type specVal struct {
ref stats.UserDefinedMetricDescriptor
valType cadvisorapiv1.DataType
time time.Time
value float64
}
udmMap := map[string]*specVal{}
for _, spec := range info.Spec.CustomMetrics {
udmMap[spec.Name] = &specVal{
ref: stats.UserDefinedMetricDescriptor{
Name: spec.Name,
Type: stats.UserDefinedMetricType(spec.Type),
Units: spec.Units,
},
valType: spec.Format,
}
}
for _, stat := range info.Stats {
for name, values := range stat.CustomMetrics {
specVal, ok := udmMap[name]
if !ok {
glog.Warningf("spec for custom metric %q is missing from cAdvisor output. Spec: %+v, Metrics: %+v", name, info.Spec, stat.CustomMetrics)
continue
}
for _, value := range values {
// Pick the most recent value
if value.Timestamp.Before(specVal.time) {
continue
}
specVal.time = value.Timestamp
specVal.value = value.FloatValue
if specVal.valType == cadvisorapiv1.IntType {
specVal.value = float64(value.IntValue)
}
}
}
}
var udm []stats.UserDefinedMetric
for _, specVal := range udmMap {
udm = append(udm, stats.UserDefinedMetric{
UserDefinedMetricDescriptor: specVal.ref,
Time: unversioned.NewTime(specVal.time),
Value: specVal.value,
})
}
return udm
}
func newPod(now time.Time, ready bool, beforeSec int) api.Pod {
conditionStatus := api.ConditionFalse
if ready {
conditionStatus = api.ConditionTrue
}
return api.Pod{
Status: api.PodStatus{
Conditions: []api.PodCondition{
{
Type: api.PodReady,
LastTransitionTime: unversioned.NewTime(now.Add(-1 * time.Duration(beforeSec) * time.Second)),
Status: conditionStatus,
},
},
},
}
}
// buildSummaryPods aggregates and returns the container stats in cinfos by the Pod managing the container.
// Containers not managed by a Pod are omitted.
func (sb *summaryBuilder) buildSummaryPods() []stats.PodStats {
// Map each container to a pod and update the PodStats with container data
podToStats := map[stats.PodReference]*stats.PodStats{}
for key, cinfo := range sb.infos {
// on systemd using devicemapper each mount into the container has an associated cgroup.
// we ignore them to ensure we do not get duplicate entries in our summary.
// for details on .mount units: http://man7.org/linux/man-pages/man5/systemd.mount.5.html
if strings.HasSuffix(key, ".mount") {
continue
}
// Build the Pod key if this container is managed by a Pod
if !sb.isPodManagedContainer(&cinfo) {
continue
}
ref := sb.buildPodRef(&cinfo)
// Lookup the PodStats for the pod using the PodRef. If none exists, initialize a new entry.
podStats, found := podToStats[ref]
if !found {
podStats = &stats.PodStats{PodRef: ref}
podToStats[ref] = podStats
}
// Update the PodStats entry with the stats from the container by adding it to stats.Containers
containerName := types.GetContainerName(cinfo.Spec.Labels)
if containerName == leaky.PodInfraContainerName {
// Special case for infrastructure container which is hidden from the user and has network stats
podStats.Network = sb.containerInfoV2ToNetworkStats("pod:"+ref.Namespace+"_"+ref.Name, &cinfo)
podStats.StartTime = unversioned.NewTime(cinfo.Spec.CreationTime)
} else {
podStats.Containers = append(podStats.Containers, sb.containerInfoV2ToStats(containerName, &cinfo))
}
}
// Add each PodStats to the result
result := make([]stats.PodStats, 0, len(podToStats))
for _, podStats := range podToStats {
// Lookup the volume stats for each pod
podUID := kubetypes.UID(podStats.PodRef.UID)
if vstats, found := sb.fsResourceAnalyzer.GetPodVolumeStats(podUID); found {
podStats.VolumeStats = vstats.Volumes
}
result = append(result, *podStats)
}
return result
}
func TestNewStatusPreservesPodStartTime(t *testing.T) {
syncer := newTestManager(&fake.Clientset{})
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
UID: "12345678",
Name: "foo",
Namespace: "new",
},
Status: api.PodStatus{},
}
now := unversioned.Now()
startTime := unversioned.NewTime(now.Time.Add(-1 * time.Minute))
pod.Status.StartTime = &startTime
syncer.SetPodStatus(pod, getRandomPodStatus())
status := expectPodStatus(t, syncer, pod)
if !status.StartTime.Time.Equal(startTime.Time) {
t.Errorf("Unexpected start time, expected %v, actual %v", startTime, status.StartTime)
}
}
func (sb *summaryBuilder) containerInfoV2ToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *stats.NetworkStats {
if !info.Spec.HasNetwork {
return nil
}
cstat, found := sb.latestContainerStats(info)
if !found {
return nil
}
for _, inter := range cstat.Network.Interfaces {
if inter.Name == network.DefaultInterfaceName {
return &stats.NetworkStats{
Time: unversioned.NewTime(cstat.Timestamp),
RxBytes: &inter.RxBytes,
RxErrors: &inter.RxErrors,
TxBytes: &inter.TxBytes,
TxErrors: &inter.TxErrors,
}
}
}
glog.Warningf("Missing default interface %q for %s", network.DefaultInterfaceName, name)
return nil
}
func TestSortableEvents(t *testing.T) {
// Arrange
list := SortableEvents([]api.Event{
{
Source: api.EventSource{Component: "kubelet"},
Message: "Item 1",
FirstTimestamp: unversioned.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)),
LastTimestamp: unversioned.NewTime(time.Date(2014, time.January, 15, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
{
Source: api.EventSource{Component: "scheduler"},
Message: "Item 2",
FirstTimestamp: unversioned.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)),
LastTimestamp: unversioned.NewTime(time.Date(1987, time.June, 17, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
{
Source: api.EventSource{Component: "kubelet"},
Message: "Item 3",
FirstTimestamp: unversioned.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)),
LastTimestamp: unversioned.NewTime(time.Date(2002, time.December, 25, 0, 0, 0, 0, time.UTC)),
Count: 1,
Type: api.EventTypeNormal,
},
})
// Act
sort.Sort(list)
// Assert
if list[0].Message != "Item 2" ||
list[1].Message != "Item 3" ||
list[2].Message != "Item 1" {
t.Fatal("List is not sorted by time. List: ", list)
}
}
// TestEventCorrelator validates proper counting, aggregation of events
func TestEventCorrelator(t *testing.T) {
firstEvent := makeEvent("first", "i am first", makeObjectReference("Pod", "my-pod", "my-ns"))
duplicateEvent := makeEvent("duplicate", "me again", makeObjectReference("Pod", "my-pod", "my-ns"))
uniqueEvent := makeEvent("unique", "snowflake", makeObjectReference("Pod", "my-pod", "my-ns"))
similarEvent := makeEvent("similar", "similar message", makeObjectReference("Pod", "my-pod", "my-ns"))
aggregateEvent := makeEvent(similarEvent.Reason, EventAggregatorByReasonMessageFunc(&similarEvent), similarEvent.InvolvedObject)
scenario := map[string]struct {
previousEvents []api.Event
newEvent api.Event
expectedEvent api.Event
intervalSeconds int
}{
"create-a-single-event": {
previousEvents: []api.Event{},
newEvent: firstEvent,
expectedEvent: setCount(firstEvent, 1),
intervalSeconds: 5,
},
"the-same-event-should-just-count": {
previousEvents: makeEvents(1, duplicateEvent),
newEvent: duplicateEvent,
expectedEvent: setCount(duplicateEvent, 2),
intervalSeconds: 5,
},
"the-same-event-should-just-count-even-if-more-than-aggregate": {
previousEvents: makeEvents(defaultAggregateMaxEvents, duplicateEvent),
newEvent: duplicateEvent,
expectedEvent: setCount(duplicateEvent, defaultAggregateMaxEvents+1),
intervalSeconds: 5,
},
"create-many-unique-events": {
previousEvents: makeUniqueEvents(30),
newEvent: uniqueEvent,
expectedEvent: setCount(uniqueEvent, 1),
intervalSeconds: 5,
},
"similar-events-should-aggregate-event": {
previousEvents: makeSimilarEvents(defaultAggregateMaxEvents-1, similarEvent, similarEvent.Message),
newEvent: similarEvent,
expectedEvent: setCount(aggregateEvent, 1),
intervalSeconds: 5,
},
"similar-events-many-times-should-count-the-aggregate": {
previousEvents: makeSimilarEvents(defaultAggregateMaxEvents, similarEvent, similarEvent.Message),
newEvent: similarEvent,
expectedEvent: setCount(aggregateEvent, 2),
intervalSeconds: 5,
},
"similar-events-whose-interval-is-greater-than-aggregate-interval-do-not-aggregate": {
previousEvents: makeSimilarEvents(defaultAggregateMaxEvents-1, similarEvent, similarEvent.Message),
newEvent: similarEvent,
expectedEvent: setCount(similarEvent, 1),
intervalSeconds: defaultAggregateIntervalInSeconds,
},
}
for testScenario, testInput := range scenario {
eventInterval := time.Duration(testInput.intervalSeconds) * time.Second
clock := util.IntervalClock{Time: time.Now(), Duration: eventInterval}
correlator := NewEventCorrelator(&clock)
for i := range testInput.previousEvents {
event := testInput.previousEvents[i]
now := unversioned.NewTime(clock.Now())
event.FirstTimestamp = now
event.LastTimestamp = now
result, err := correlator.EventCorrelate(&event)
if err != nil {
t.Errorf("scenario %v: unexpected error playing back prevEvents %v", testScenario, err)
}
correlator.UpdateState(result.Event)
}
// update the input to current clock value
now := unversioned.NewTime(clock.Now())
testInput.newEvent.FirstTimestamp = now
testInput.newEvent.LastTimestamp = now
result, err := correlator.EventCorrelate(&testInput.newEvent)
if err != nil {
t.Errorf("scenario %v: unexpected error correlating input event %v", testScenario, err)
}
_, err = validateEvent(testScenario, result.Event, &testInput.expectedEvent, t)
if err != nil {
t.Errorf("scenario %v: unexpected error validating result %v", testScenario, err)
}
}
}