func addPodInfo(key string, podMs *core.MetricSet, pod *kube_api.Pod, batch *core.DataBatch, newMs map[string]*core.MetricSet) { // Add UID to pod podMs.Labels[core.LabelPodId.Key] = string(pod.UID) podMs.Labels[core.LabelLabels.Key] = util.LabelsToString(pod.Labels, ",") // Add cpu/mem requests and limits to containers for _, container := range pod.Spec.Containers { containerKey := core.PodContainerKey(pod.Namespace, pod.Name, container.Name) if _, found := batch.MetricSets[containerKey]; !found { if _, found := newMs[containerKey]; !found { glog.V(2).Infof("Container %s not found, creating a stub", containerKey) containerMs := &core.MetricSet{ MetricValues: make(map[string]core.MetricValue), Labels: map[string]string{ core.LabelMetricSetType.Key: core.MetricSetTypePodContainer, core.LabelNamespaceName.Key: pod.Namespace, core.LabelPodNamespace.Key: pod.Namespace, core.LabelPodName.Key: pod.Name, core.LabelContainerName.Key: container.Name, core.LabelContainerBaseImage.Key: container.Image, core.LabelPodId.Key: string(pod.UID), core.LabelLabels.Key: util.LabelsToString(pod.Labels, ","), core.LabelNodename.Key: podMs.Labels[core.LabelNodename.Key], core.LabelHostname.Key: podMs.Labels[core.LabelHostname.Key], core.LabelHostID.Key: podMs.Labels[core.LabelHostID.Key], }, } updateContainerResourcesAndLimits(containerMs, container) newMs[containerKey] = containerMs } } } }
// availableMetrics returns a list of available pod metric names. func (a *Api) availablePodContainerMetrics(request *restful.Request, response *restful.Response) { a.processMetricNamesRequest( core.PodContainerKey(request.PathParameter("namespace-name"), request.PathParameter("pod-name"), request.PathParameter("container-name"), ), response) }
func (m *MetricStorage) getPodMetrics(pod *api.Pod) *metrics.PodMetrics { batch := m.metricSink.GetLatestDataBatch() if batch == nil { return nil } res := &metrics.PodMetrics{ ObjectMeta: api.ObjectMeta{ Name: pod.Name, Namespace: pod.Namespace, CreationTimestamp: unversioned.NewTime(time.Now()), }, Timestamp: unversioned.NewTime(batch.Timestamp), Window: unversioned.Duration{Duration: time.Minute}, Containers: make([]metrics.ContainerMetrics, 0), } for _, c := range pod.Spec.Containers { ms, found := batch.MetricSets[core.PodContainerKey(pod.Namespace, pod.Name, c.Name)] if !found { glog.Infof("No metrics for container %s in pod %s/%s", c.Name, pod.Namespace, pod.Name) return nil } usage, err := util.ParseResourceList(ms) if err != nil { return nil } res.Containers = append(res.Containers, metrics.ContainerMetrics{Name: c.Name, Usage: usage}) } return res }
func addContainerInfo(key string, containerMs *core.MetricSet, pod *kube_api.Pod, batch *core.DataBatch, newMs map[string]*core.MetricSet) { for _, container := range pod.Spec.Containers { if key == core.PodContainerKey(pod.Namespace, pod.Name, container.Name) { updateContainerResourcesAndLimits(containerMs, container) if _, ok := containerMs.Labels[core.LabelContainerBaseImage.Key]; !ok { containerMs.Labels[core.LabelContainerBaseImage.Key] = container.Image } break } } containerMs.Labels[core.LabelPodId.Key] = string(pod.UID) containerMs.Labels[core.LabelLabels.Key] = util.LabelsToString(pod.Labels, ",") namespace := containerMs.Labels[core.LabelNamespaceName.Key] podName := containerMs.Labels[core.LabelPodName.Key] podKey := core.PodKey(namespace, podName) _, oldfound := batch.MetricSets[podKey] if !oldfound { _, newfound := newMs[podKey] if !newfound { glog.V(2).Infof("Pod %s not found, creating a stub", podKey) podMs := &core.MetricSet{ MetricValues: make(map[string]core.MetricValue), Labels: map[string]string{ core.LabelMetricSetType.Key: core.MetricSetTypePod, core.LabelNamespaceName.Key: namespace, core.LabelPodNamespace.Key: namespace, core.LabelPodName.Key: podName, core.LabelNodename.Key: containerMs.Labels[core.LabelNodename.Key], core.LabelHostname.Key: containerMs.Labels[core.LabelHostname.Key], core.LabelHostID.Key: containerMs.Labels[core.LabelHostID.Key], }, } newMs[podKey] = podMs addPodInfo(podKey, podMs, pod, batch, newMs) } } }
func TestPodEnricher(t *testing.T) { pod := kube_api.Pod{ ObjectMeta: kube_api.ObjectMeta{ Name: "pod1", Namespace: "ns1", }, Spec: kube_api.PodSpec{ NodeName: "node1", Containers: []kube_api.Container{ { Name: "c1", Image: "gcr.io/google_containers/pause:2.0", Resources: kube_api.ResourceRequirements{ Requests: kube_api.ResourceList{ kube_api.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), kube_api.ResourceMemory: *resource.NewQuantity(555, resource.DecimalSI), }, }, }, { Name: "nginx", Image: "gcr.io/google_containers/pause:2.0", Resources: kube_api.ResourceRequirements{ Requests: kube_api.ResourceList{ kube_api.ResourceCPU: *resource.NewMilliQuantity(333, resource.DecimalSI), kube_api.ResourceMemory: *resource.NewQuantity(1000, resource.DecimalSI), }, Limits: kube_api.ResourceList{ kube_api.ResourceCPU: *resource.NewMilliQuantity(2222, resource.DecimalSI), kube_api.ResourceMemory: *resource.NewQuantity(3333, resource.DecimalSI), }, }, }, }, }, } store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) podLister := &cache.StoreToPodLister{Indexer: store} podLister.Indexer.Add(&pod) podBasedEnricher := PodBasedEnricher{podLister: podLister} var err error for _, batch := range batches { batch, err = podBasedEnricher.Process(batch) assert.NoError(t, err) podAggregator := PodAggregator{} batch, err = podAggregator.Process(batch) assert.NoError(t, err) podMs, found := batch.MetricSets[core.PodKey("ns1", "pod1")] assert.True(t, found) checkRequests(t, podMs, 433, 1555) checkLimits(t, podMs, 2222, 3333) containerMs, found := batch.MetricSets[core.PodContainerKey("ns1", "pod1", "c1")] assert.True(t, found) checkRequests(t, containerMs, 100, 555) checkLimits(t, containerMs, 0, 0) } }
"time" "github.com/stretchr/testify/assert" "k8s.io/heapster/metrics/core" kube_api "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/client/cache" ) var batches = []*core.DataBatch{ &core.DataBatch{ Timestamp: time.Now(), MetricSets: map[string]*core.MetricSet{ core.PodContainerKey("ns1", "pod1", "c1"): { Labels: map[string]string{ core.LabelMetricSetType.Key: core.MetricSetTypePodContainer, core.LabelPodName.Key: "pod1", core.LabelNamespaceName.Key: "ns1", core.LabelContainerName.Key: "c1", }, MetricValues: map[string]core.MetricValue{}, }, core.PodKey("ns1", "pod1"): { Labels: map[string]string{ core.LabelMetricSetType.Key: core.MetricSetTypePod, core.LabelPodName.Key: "pod1", core.LabelNamespaceName.Key: "ns1", },
func TestDecodeSummaryMetrics(t *testing.T) { ms := testingSummaryMetricsSource() summary := stats.Summary{ Node: stats.NodeStats{ NodeName: nodeInfo.NodeName, StartTime: unversioned.NewTime(startTime), CPU: genTestSummaryCPU(seedNode), Memory: genTestSummaryMemory(seedNode), Network: genTestSummaryNetwork(seedNode), SystemContainers: []stats.ContainerStats{ genTestSummaryContainer(stats.SystemContainerKubelet, seedKubelet), genTestSummaryContainer(stats.SystemContainerRuntime, seedRuntime), genTestSummaryContainer(stats.SystemContainerMisc, seedMisc), }, Fs: genTestSummaryFsStats(seedNode), }, Pods: []stats.PodStats{{ PodRef: stats.PodReference{ Name: pName0, Namespace: namespace0, }, StartTime: unversioned.NewTime(startTime), Network: genTestSummaryNetwork(seedPod0), Containers: []stats.ContainerStats{ genTestSummaryContainer(cName00, seedPod0Container0), genTestSummaryContainer(cName01, seedPod0Container1), }, }, { PodRef: stats.PodReference{ Name: pName1, Namespace: namespace0, }, StartTime: unversioned.NewTime(startTime), Network: genTestSummaryNetwork(seedPod1), Containers: []stats.ContainerStats{ genTestSummaryContainer(cName10, seedPod1Container), }, VolumeStats: []stats.VolumeStats{{ Name: "A", FsStats: *genTestSummaryFsStats(seedPod1), }, { Name: "B", FsStats: *genTestSummaryFsStats(seedPod1), }}, }, { PodRef: stats.PodReference{ Name: pName2, Namespace: namespace1, }, StartTime: unversioned.NewTime(startTime), Network: genTestSummaryNetwork(seedPod2), Containers: []stats.ContainerStats{ genTestSummaryContainer(cName20, seedPod2Container), }, }}, } containerFs := []string{"/", "logs"} expectations := []struct { key string setType string seed int64 cpu bool memory bool network bool fs []string }{{ key: core.NodeKey(nodeInfo.NodeName), setType: core.MetricSetTypeNode, seed: seedNode, cpu: true, memory: true, network: true, fs: []string{"/"}, }, { key: core.NodeContainerKey(nodeInfo.NodeName, "kubelet"), setType: core.MetricSetTypeSystemContainer, seed: seedKubelet, cpu: true, memory: true, }, { key: core.NodeContainerKey(nodeInfo.NodeName, "docker-daemon"), setType: core.MetricSetTypeSystemContainer, seed: seedRuntime, cpu: true, memory: true, }, { key: core.NodeContainerKey(nodeInfo.NodeName, "system"), setType: core.MetricSetTypeSystemContainer, seed: seedMisc, cpu: true, memory: true, }, { key: core.PodKey(namespace0, pName0), setType: core.MetricSetTypePod, seed: seedPod0, network: true, }, { key: core.PodKey(namespace0, pName1), setType: core.MetricSetTypePod, seed: seedPod1, network: true, fs: []string{"Volume:A", "Volume:B"}, }, { key: core.PodKey(namespace1, pName2), setType: core.MetricSetTypePod, seed: seedPod2, network: true, }, { key: core.PodContainerKey(namespace0, pName0, cName00), setType: core.MetricSetTypePodContainer, seed: seedPod0Container0, cpu: true, memory: true, fs: containerFs, }, { key: core.PodContainerKey(namespace0, pName0, cName01), setType: core.MetricSetTypePodContainer, seed: seedPod0Container1, cpu: true, memory: true, fs: containerFs, }, { key: core.PodContainerKey(namespace0, pName1, cName10), setType: core.MetricSetTypePodContainer, seed: seedPod1Container, cpu: true, memory: true, fs: containerFs, }, { key: core.PodContainerKey(namespace1, pName2, cName20), setType: core.MetricSetTypePodContainer, seed: seedPod2Container, cpu: true, memory: true, fs: containerFs, }} metrics := ms.decodeSummary(&summary) for _, e := range expectations { m, ok := metrics[e.key] if !assert.True(t, ok, "missing metric %q", e.key) { continue } assert.Equal(t, m.Labels[core.LabelMetricSetType.Key], e.setType, e.key) assert.Equal(t, m.CreateTime, startTime, e.key) assert.Equal(t, m.ScrapeTime, scrapeTime, e.key) if e.cpu { checkIntMetric(t, m, e.key, core.MetricCpuUsage, e.seed+offsetCPUUsageCoreSeconds) } if e.memory { checkIntMetric(t, m, e.key, core.MetricMemoryUsage, e.seed+offsetMemUsageBytes) checkIntMetric(t, m, e.key, core.MetricMemoryWorkingSet, e.seed+offsetMemWorkingSetBytes) checkIntMetric(t, m, e.key, core.MetricMemoryPageFaults, e.seed+offsetMemPageFaults) checkIntMetric(t, m, e.key, core.MetricMemoryMajorPageFaults, e.seed+offsetMemMajorPageFaults) } if e.network { checkIntMetric(t, m, e.key, core.MetricNetworkRx, e.seed+offsetNetRxBytes) checkIntMetric(t, m, e.key, core.MetricNetworkRxErrors, e.seed+offsetNetRxErrors) checkIntMetric(t, m, e.key, core.MetricNetworkTx, e.seed+offsetNetTxBytes) checkIntMetric(t, m, e.key, core.MetricNetworkTxErrors, e.seed+offsetNetTxErrors) } for _, label := range e.fs { checkFsMetric(t, m, e.key, label, core.MetricFilesystemAvailable, e.seed+offsetFsAvailable) checkFsMetric(t, m, e.key, label, core.MetricFilesystemLimit, e.seed+offsetFsCapacity) checkFsMetric(t, m, e.key, label, core.MetricFilesystemUsage, e.seed+offsetFsUsed) } delete(metrics, e.key) } for k, v := range metrics { assert.Fail(t, "unexpected metric", "%q: %+v", k, v) } }
func TestPodAggregator(t *testing.T) { batch := core.DataBatch{ Timestamp: time.Now(), MetricSets: map[string]*core.MetricSet{ core.PodContainerKey("ns1", "pod1", "c1"): { Labels: map[string]string{ core.LabelMetricSetType.Key: core.MetricSetTypePodContainer, core.LabelPodName.Key: "pod1", core.LabelNamespaceName.Key: "ns1", }, MetricValues: map[string]core.MetricValue{ "m1": { ValueType: core.ValueInt64, MetricType: core.MetricGauge, IntValue: 10, }, "m2": { ValueType: core.ValueInt64, MetricType: core.MetricGauge, IntValue: 222, }, }, }, core.PodContainerKey("ns1", "pod1", "c2"): { Labels: map[string]string{ core.LabelMetricSetType.Key: core.MetricSetTypePodContainer, core.LabelPodName.Key: "pod1", core.LabelNamespaceName.Key: "ns1", }, MetricValues: map[string]core.MetricValue{ "m1": { ValueType: core.ValueInt64, MetricType: core.MetricGauge, IntValue: 100, }, "m3": { ValueType: core.ValueInt64, MetricType: core.MetricGauge, IntValue: 30, }, }, }, }, } processor := PodAggregator{} result, err := processor.Process(&batch) assert.NoError(t, err) pod, found := result.MetricSets[core.PodKey("ns1", "pod1")] assert.True(t, found) m1, found := pod.MetricValues["m1"] assert.True(t, found) assert.Equal(t, int64(110), m1.IntValue) m2, found := pod.MetricValues["m2"] assert.True(t, found) assert.Equal(t, int64(222), m2.IntValue) m3, found := pod.MetricValues["m3"] assert.True(t, found) assert.Equal(t, int64(30), m3.IntValue) labelPodName, found := pod.Labels[core.LabelPodName.Key] assert.True(t, found) assert.Equal(t, "pod1", labelPodName) labelNsName, found := pod.Labels[core.LabelNamespaceName.Key] assert.True(t, found) assert.Equal(t, "ns1", labelNsName) }
func TestRateCalculator(t *testing.T) { key := core.PodContainerKey("ns1", "pod1", "c") now := time.Now() prev := &core.DataBatch{ Timestamp: now.Add(-time.Minute), MetricSets: map[string]*core.MetricSet{ key: { CreateTime: now.Add(-time.Hour), ScrapeTime: now.Add(-60 * time.Second), Labels: map[string]string{ core.LabelMetricSetType.Key: core.MetricSetTypePodContainer, }, MetricValues: map[string]core.MetricValue{ core.MetricCpuUsage.MetricDescriptor.Name: { ValueType: core.ValueInt64, MetricType: core.MetricCumulative, IntValue: 947130377781, }, core.MetricNetworkTxErrors.MetricDescriptor.Name: { ValueType: core.ValueInt64, MetricType: core.MetricCumulative, IntValue: 0, }, }, }, }, } current := &core.DataBatch{ Timestamp: now, MetricSets: map[string]*core.MetricSet{ key: { CreateTime: now.Add(-time.Hour), ScrapeTime: now, Labels: map[string]string{ core.LabelMetricSetType.Key: core.MetricSetTypePodContainer, }, MetricValues: map[string]core.MetricValue{ core.MetricCpuUsage.MetricDescriptor.Name: { ValueType: core.ValueInt64, MetricType: core.MetricCumulative, IntValue: 948071062732, }, core.MetricNetworkTxErrors.MetricDescriptor.Name: { ValueType: core.ValueInt64, MetricType: core.MetricCumulative, IntValue: 120, }, }, }, }, } procesor := NewRateCalculator(core.RateMetricsMapping) procesor.Process(prev) procesor.Process(current) ms := current.MetricSets[key] cpuRate := ms.MetricValues[core.MetricCpuUsageRate.Name] txeRate := ms.MetricValues[core.MetricNetworkTxErrorsRate.Name] assert.InEpsilon(t, 13, cpuRate.IntValue, 2) assert.InEpsilon(t, 2, txeRate.FloatValue, 0.1) }