func addPodInfo(key string, podMs *core.MetricSet, pod *kube_api.Pod, batch *core.DataBatch, newMs map[string]*core.MetricSet) {
// Add UID to pod
podMs.Labels[core.LabelPodId.Key] = string(pod.UID)
podMs.Labels[core.LabelLabels.Key] = util.LabelsToString(pod.Labels, ",")
// Add cpu/mem requests and limits to containers
for _, container := range pod.Spec.Containers {
containerKey := core.PodContainerKey(pod.Namespace, pod.Name, container.Name)
if _, found := batch.MetricSets[containerKey]; !found {
if _, found := newMs[containerKey]; !found {
glog.V(2).Infof("Container %s not found, creating a stub", containerKey)
containerMs := &core.MetricSet{
MetricValues: make(map[string]core.MetricValue),
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
core.LabelNamespaceName.Key: pod.Namespace,
core.LabelPodNamespace.Key: pod.Namespace,
core.LabelPodName.Key: pod.Name,
core.LabelContainerName.Key: container.Name,
core.LabelContainerBaseImage.Key: container.Image,
core.LabelPodId.Key: string(pod.UID),
core.LabelLabels.Key: util.LabelsToString(pod.Labels, ","),
core.LabelNodename.Key: podMs.Labels[core.LabelNodename.Key],
core.LabelHostname.Key: podMs.Labels[core.LabelHostname.Key],
core.LabelHostID.Key: podMs.Labels[core.LabelHostID.Key],
},
}
updateContainerResourcesAndLimits(containerMs, container)
newMs[containerKey] = containerMs
}
}
}
}
// availableMetrics returns a list of available pod metric names.
func (a *Api) availablePodContainerMetrics(request *restful.Request, response *restful.Response) {
a.processMetricNamesRequest(
core.PodContainerKey(request.PathParameter("namespace-name"),
request.PathParameter("pod-name"),
request.PathParameter("container-name"),
), response)
}
func (m *MetricStorage) getPodMetrics(pod *api.Pod) *metrics.PodMetrics {
batch := m.metricSink.GetLatestDataBatch()
if batch == nil {
return nil
}
res := &metrics.PodMetrics{
ObjectMeta: api.ObjectMeta{
Name: pod.Name,
Namespace: pod.Namespace,
CreationTimestamp: unversioned.NewTime(time.Now()),
},
Timestamp: unversioned.NewTime(batch.Timestamp),
Window: unversioned.Duration{Duration: time.Minute},
Containers: make([]metrics.ContainerMetrics, 0),
}
for _, c := range pod.Spec.Containers {
ms, found := batch.MetricSets[core.PodContainerKey(pod.Namespace, pod.Name, c.Name)]
if !found {
glog.Infof("No metrics for container %s in pod %s/%s", c.Name, pod.Namespace, pod.Name)
return nil
}
usage, err := util.ParseResourceList(ms)
if err != nil {
return nil
}
res.Containers = append(res.Containers, metrics.ContainerMetrics{Name: c.Name, Usage: usage})
}
return res
}
func addContainerInfo(key string, containerMs *core.MetricSet, pod *kube_api.Pod, batch *core.DataBatch, newMs map[string]*core.MetricSet) {
for _, container := range pod.Spec.Containers {
if key == core.PodContainerKey(pod.Namespace, pod.Name, container.Name) {
updateContainerResourcesAndLimits(containerMs, container)
if _, ok := containerMs.Labels[core.LabelContainerBaseImage.Key]; !ok {
containerMs.Labels[core.LabelContainerBaseImage.Key] = container.Image
}
break
}
}
containerMs.Labels[core.LabelPodId.Key] = string(pod.UID)
containerMs.Labels[core.LabelLabels.Key] = util.LabelsToString(pod.Labels, ",")
namespace := containerMs.Labels[core.LabelNamespaceName.Key]
podName := containerMs.Labels[core.LabelPodName.Key]
podKey := core.PodKey(namespace, podName)
_, oldfound := batch.MetricSets[podKey]
if !oldfound {
_, newfound := newMs[podKey]
if !newfound {
glog.V(2).Infof("Pod %s not found, creating a stub", podKey)
podMs := &core.MetricSet{
MetricValues: make(map[string]core.MetricValue),
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePod,
core.LabelNamespaceName.Key: namespace,
core.LabelPodNamespace.Key: namespace,
core.LabelPodName.Key: podName,
core.LabelNodename.Key: containerMs.Labels[core.LabelNodename.Key],
core.LabelHostname.Key: containerMs.Labels[core.LabelHostname.Key],
core.LabelHostID.Key: containerMs.Labels[core.LabelHostID.Key],
},
}
newMs[podKey] = podMs
addPodInfo(podKey, podMs, pod, batch, newMs)
}
}
}
func TestPodEnricher(t *testing.T) {
pod := kube_api.Pod{
ObjectMeta: kube_api.ObjectMeta{
Name: "pod1",
Namespace: "ns1",
},
Spec: kube_api.PodSpec{
NodeName: "node1",
Containers: []kube_api.Container{
{
Name: "c1",
Image: "gcr.io/google_containers/pause:2.0",
Resources: kube_api.ResourceRequirements{
Requests: kube_api.ResourceList{
kube_api.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI),
kube_api.ResourceMemory: *resource.NewQuantity(555, resource.DecimalSI),
},
},
},
{
Name: "nginx",
Image: "gcr.io/google_containers/pause:2.0",
Resources: kube_api.ResourceRequirements{
Requests: kube_api.ResourceList{
kube_api.ResourceCPU: *resource.NewMilliQuantity(333, resource.DecimalSI),
kube_api.ResourceMemory: *resource.NewQuantity(1000, resource.DecimalSI),
},
Limits: kube_api.ResourceList{
kube_api.ResourceCPU: *resource.NewMilliQuantity(2222, resource.DecimalSI),
kube_api.ResourceMemory: *resource.NewQuantity(3333, resource.DecimalSI),
},
},
},
},
},
}
store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
podLister := &cache.StoreToPodLister{Indexer: store}
podLister.Indexer.Add(&pod)
podBasedEnricher := PodBasedEnricher{podLister: podLister}
var err error
for _, batch := range batches {
batch, err = podBasedEnricher.Process(batch)
assert.NoError(t, err)
podAggregator := PodAggregator{}
batch, err = podAggregator.Process(batch)
assert.NoError(t, err)
podMs, found := batch.MetricSets[core.PodKey("ns1", "pod1")]
assert.True(t, found)
checkRequests(t, podMs, 433, 1555)
checkLimits(t, podMs, 2222, 3333)
containerMs, found := batch.MetricSets[core.PodContainerKey("ns1", "pod1", "c1")]
assert.True(t, found)
checkRequests(t, containerMs, 100, 555)
checkLimits(t, containerMs, 0, 0)
}
}
"time"
"github.com/stretchr/testify/assert"
"k8s.io/heapster/metrics/core"
kube_api "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
"k8s.io/kubernetes/pkg/client/cache"
)
var batches = []*core.DataBatch{
&core.DataBatch{
Timestamp: time.Now(),
MetricSets: map[string]*core.MetricSet{
core.PodContainerKey("ns1", "pod1", "c1"): {
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
core.LabelPodName.Key: "pod1",
core.LabelNamespaceName.Key: "ns1",
core.LabelContainerName.Key: "c1",
},
MetricValues: map[string]core.MetricValue{},
},
core.PodKey("ns1", "pod1"): {
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePod,
core.LabelPodName.Key: "pod1",
core.LabelNamespaceName.Key: "ns1",
},
func TestDecodeSummaryMetrics(t *testing.T) {
ms := testingSummaryMetricsSource()
summary := stats.Summary{
Node: stats.NodeStats{
NodeName: nodeInfo.NodeName,
StartTime: unversioned.NewTime(startTime),
CPU: genTestSummaryCPU(seedNode),
Memory: genTestSummaryMemory(seedNode),
Network: genTestSummaryNetwork(seedNode),
SystemContainers: []stats.ContainerStats{
genTestSummaryContainer(stats.SystemContainerKubelet, seedKubelet),
genTestSummaryContainer(stats.SystemContainerRuntime, seedRuntime),
genTestSummaryContainer(stats.SystemContainerMisc, seedMisc),
},
Fs: genTestSummaryFsStats(seedNode),
},
Pods: []stats.PodStats{{
PodRef: stats.PodReference{
Name: pName0,
Namespace: namespace0,
},
StartTime: unversioned.NewTime(startTime),
Network: genTestSummaryNetwork(seedPod0),
Containers: []stats.ContainerStats{
genTestSummaryContainer(cName00, seedPod0Container0),
genTestSummaryContainer(cName01, seedPod0Container1),
},
}, {
PodRef: stats.PodReference{
Name: pName1,
Namespace: namespace0,
},
StartTime: unversioned.NewTime(startTime),
Network: genTestSummaryNetwork(seedPod1),
Containers: []stats.ContainerStats{
genTestSummaryContainer(cName10, seedPod1Container),
},
VolumeStats: []stats.VolumeStats{{
Name: "A",
FsStats: *genTestSummaryFsStats(seedPod1),
}, {
Name: "B",
FsStats: *genTestSummaryFsStats(seedPod1),
}},
}, {
PodRef: stats.PodReference{
Name: pName2,
Namespace: namespace1,
},
StartTime: unversioned.NewTime(startTime),
Network: genTestSummaryNetwork(seedPod2),
Containers: []stats.ContainerStats{
genTestSummaryContainer(cName20, seedPod2Container),
},
}},
}
containerFs := []string{"/", "logs"}
expectations := []struct {
key string
setType string
seed int64
cpu bool
memory bool
network bool
fs []string
}{{
key: core.NodeKey(nodeInfo.NodeName),
setType: core.MetricSetTypeNode,
seed: seedNode,
cpu: true,
memory: true,
network: true,
fs: []string{"/"},
}, {
key: core.NodeContainerKey(nodeInfo.NodeName, "kubelet"),
setType: core.MetricSetTypeSystemContainer,
seed: seedKubelet,
cpu: true,
memory: true,
}, {
key: core.NodeContainerKey(nodeInfo.NodeName, "docker-daemon"),
setType: core.MetricSetTypeSystemContainer,
seed: seedRuntime,
cpu: true,
memory: true,
}, {
key: core.NodeContainerKey(nodeInfo.NodeName, "system"),
setType: core.MetricSetTypeSystemContainer,
seed: seedMisc,
cpu: true,
memory: true,
}, {
key: core.PodKey(namespace0, pName0),
setType: core.MetricSetTypePod,
seed: seedPod0,
network: true,
}, {
key: core.PodKey(namespace0, pName1),
setType: core.MetricSetTypePod,
seed: seedPod1,
network: true,
fs: []string{"Volume:A", "Volume:B"},
}, {
key: core.PodKey(namespace1, pName2),
setType: core.MetricSetTypePod,
seed: seedPod2,
network: true,
}, {
key: core.PodContainerKey(namespace0, pName0, cName00),
setType: core.MetricSetTypePodContainer,
seed: seedPod0Container0,
cpu: true,
memory: true,
fs: containerFs,
}, {
key: core.PodContainerKey(namespace0, pName0, cName01),
setType: core.MetricSetTypePodContainer,
seed: seedPod0Container1,
cpu: true,
memory: true,
fs: containerFs,
}, {
key: core.PodContainerKey(namespace0, pName1, cName10),
setType: core.MetricSetTypePodContainer,
seed: seedPod1Container,
cpu: true,
memory: true,
fs: containerFs,
}, {
key: core.PodContainerKey(namespace1, pName2, cName20),
setType: core.MetricSetTypePodContainer,
seed: seedPod2Container,
cpu: true,
memory: true,
fs: containerFs,
}}
metrics := ms.decodeSummary(&summary)
for _, e := range expectations {
m, ok := metrics[e.key]
if !assert.True(t, ok, "missing metric %q", e.key) {
continue
}
assert.Equal(t, m.Labels[core.LabelMetricSetType.Key], e.setType, e.key)
assert.Equal(t, m.CreateTime, startTime, e.key)
assert.Equal(t, m.ScrapeTime, scrapeTime, e.key)
if e.cpu {
checkIntMetric(t, m, e.key, core.MetricCpuUsage, e.seed+offsetCPUUsageCoreSeconds)
}
if e.memory {
checkIntMetric(t, m, e.key, core.MetricMemoryUsage, e.seed+offsetMemUsageBytes)
checkIntMetric(t, m, e.key, core.MetricMemoryWorkingSet, e.seed+offsetMemWorkingSetBytes)
checkIntMetric(t, m, e.key, core.MetricMemoryPageFaults, e.seed+offsetMemPageFaults)
checkIntMetric(t, m, e.key, core.MetricMemoryMajorPageFaults, e.seed+offsetMemMajorPageFaults)
}
if e.network {
checkIntMetric(t, m, e.key, core.MetricNetworkRx, e.seed+offsetNetRxBytes)
checkIntMetric(t, m, e.key, core.MetricNetworkRxErrors, e.seed+offsetNetRxErrors)
checkIntMetric(t, m, e.key, core.MetricNetworkTx, e.seed+offsetNetTxBytes)
checkIntMetric(t, m, e.key, core.MetricNetworkTxErrors, e.seed+offsetNetTxErrors)
}
for _, label := range e.fs {
checkFsMetric(t, m, e.key, label, core.MetricFilesystemAvailable, e.seed+offsetFsAvailable)
checkFsMetric(t, m, e.key, label, core.MetricFilesystemLimit, e.seed+offsetFsCapacity)
checkFsMetric(t, m, e.key, label, core.MetricFilesystemUsage, e.seed+offsetFsUsed)
}
delete(metrics, e.key)
}
for k, v := range metrics {
assert.Fail(t, "unexpected metric", "%q: %+v", k, v)
}
}
func TestPodAggregator(t *testing.T) {
batch := core.DataBatch{
Timestamp: time.Now(),
MetricSets: map[string]*core.MetricSet{
core.PodContainerKey("ns1", "pod1", "c1"): {
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
core.LabelPodName.Key: "pod1",
core.LabelNamespaceName.Key: "ns1",
},
MetricValues: map[string]core.MetricValue{
"m1": {
ValueType: core.ValueInt64,
MetricType: core.MetricGauge,
IntValue: 10,
},
"m2": {
ValueType: core.ValueInt64,
MetricType: core.MetricGauge,
IntValue: 222,
},
},
},
core.PodContainerKey("ns1", "pod1", "c2"): {
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
core.LabelPodName.Key: "pod1",
core.LabelNamespaceName.Key: "ns1",
},
MetricValues: map[string]core.MetricValue{
"m1": {
ValueType: core.ValueInt64,
MetricType: core.MetricGauge,
IntValue: 100,
},
"m3": {
ValueType: core.ValueInt64,
MetricType: core.MetricGauge,
IntValue: 30,
},
},
},
},
}
processor := PodAggregator{}
result, err := processor.Process(&batch)
assert.NoError(t, err)
pod, found := result.MetricSets[core.PodKey("ns1", "pod1")]
assert.True(t, found)
m1, found := pod.MetricValues["m1"]
assert.True(t, found)
assert.Equal(t, int64(110), m1.IntValue)
m2, found := pod.MetricValues["m2"]
assert.True(t, found)
assert.Equal(t, int64(222), m2.IntValue)
m3, found := pod.MetricValues["m3"]
assert.True(t, found)
assert.Equal(t, int64(30), m3.IntValue)
labelPodName, found := pod.Labels[core.LabelPodName.Key]
assert.True(t, found)
assert.Equal(t, "pod1", labelPodName)
labelNsName, found := pod.Labels[core.LabelNamespaceName.Key]
assert.True(t, found)
assert.Equal(t, "ns1", labelNsName)
}
func TestRateCalculator(t *testing.T) {
key := core.PodContainerKey("ns1", "pod1", "c")
now := time.Now()
prev := &core.DataBatch{
Timestamp: now.Add(-time.Minute),
MetricSets: map[string]*core.MetricSet{
key: {
CreateTime: now.Add(-time.Hour),
ScrapeTime: now.Add(-60 * time.Second),
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
},
MetricValues: map[string]core.MetricValue{
core.MetricCpuUsage.MetricDescriptor.Name: {
ValueType: core.ValueInt64,
MetricType: core.MetricCumulative,
IntValue: 947130377781,
},
core.MetricNetworkTxErrors.MetricDescriptor.Name: {
ValueType: core.ValueInt64,
MetricType: core.MetricCumulative,
IntValue: 0,
},
},
},
},
}
current := &core.DataBatch{
Timestamp: now,
MetricSets: map[string]*core.MetricSet{
key: {
CreateTime: now.Add(-time.Hour),
ScrapeTime: now,
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
},
MetricValues: map[string]core.MetricValue{
core.MetricCpuUsage.MetricDescriptor.Name: {
ValueType: core.ValueInt64,
MetricType: core.MetricCumulative,
IntValue: 948071062732,
},
core.MetricNetworkTxErrors.MetricDescriptor.Name: {
ValueType: core.ValueInt64,
MetricType: core.MetricCumulative,
IntValue: 120,
},
},
},
},
}
procesor := NewRateCalculator(core.RateMetricsMapping)
procesor.Process(prev)
procesor.Process(current)
ms := current.MetricSets[key]
cpuRate := ms.MetricValues[core.MetricCpuUsageRate.Name]
txeRate := ms.MetricValues[core.MetricNetworkTxErrorsRate.Name]
assert.InEpsilon(t, 13, cpuRate.IntValue, 2)
assert.InEpsilon(t, 2, txeRate.FloatValue, 0.1)
}