func (m *MetricStorage) getNodeMetrics(node string) *metrics.NodeMetrics {
batch := m.metricSink.GetLatestDataBatch()
if batch == nil {
return nil
}
ms, found := batch.MetricSets[core.NodeKey(node)]
if !found {
return nil
}
usage, err := util.ParseResourceList(ms)
if err != nil {
return nil
}
return &metrics.NodeMetrics{
ObjectMeta: api.ObjectMeta{
Name: node,
CreationTimestamp: unversioned.NewTime(time.Now()),
},
Timestamp: unversioned.NewTime(batch.Timestamp),
Window: unversioned.Duration{Duration: time.Minute},
Usage: usage,
}
}
func (this *NodeAutoscalingEnricher) Process(batch *core.DataBatch) (*core.DataBatch, error) {
nodes, err := this.nodeLister.List()
if err != nil {
return nil, err
}
for _, node := range nodes.Items {
if metricSet, found := batch.MetricSets[core.NodeKey(node.Name)]; found {
metricSet.Labels[core.LabelLabels.Key] = util.LabelsToString(node.Labels, ",")
availableCpu, _ := node.Status.Capacity[kube_api.ResourceCPU]
availableMem, _ := node.Status.Capacity[kube_api.ResourceMemory]
cpuRequested := getInt(metricSet, &core.MetricCpuRequest)
cpuUsed := getInt(metricSet, &core.MetricCpuUsageRate)
memRequested := getInt(metricSet, &core.MetricMemoryRequest)
memUsed := getInt(metricSet, &core.MetricMemoryUsage)
if availableCpu.MilliValue() != 0 {
setFloat(metricSet, &core.MetricNodeCpuUtilization, float32(cpuUsed)/float32(availableCpu.MilliValue()))
setFloat(metricSet, &core.MetricNodeCpuReservation, float32(cpuRequested)/float32(availableCpu.MilliValue()))
setFloat(metricSet, &core.MetricNodeCpuCapacity, float32(availableCpu.MilliValue()))
}
if availableMem.Value() != 0 {
setFloat(metricSet, &core.MetricNodeMemoryUtilization, float32(memUsed)/float32(availableMem.Value()))
setFloat(metricSet, &core.MetricNodeMemoryReservation, float32(memRequested)/float32(availableMem.Value()))
setFloat(metricSet, &core.MetricNodeMemoryCapacity, float32(availableMem.Value()))
}
}
}
return batch, nil
}
func (a *Api) getNodeMetrics(node string) *v1alpha1.NodeMetrics {
batch := a.metricSink.GetLatestDataBatch()
if batch == nil {
return nil
}
ms, found := batch.MetricSets[core.NodeKey(node)]
if !found {
return nil
}
usage, err := parseResourceList(ms)
if err != nil {
return nil
}
return &v1alpha1.NodeMetrics{
ObjectMeta: kube_v1.ObjectMeta{
Name: node,
CreationTimestamp: kube_unversioned.NewTime(time.Now()),
},
Timestamp: kube_unversioned.NewTime(batch.Timestamp),
Window: kube_unversioned.Duration{Duration: time.Minute},
Usage: usage,
}
}
func (this *NodeAggregator) Process(batch *core.DataBatch) (*core.DataBatch, error) {
for key, metricSet := range batch.MetricSets {
if metricSetType, found := metricSet.Labels[core.LabelMetricSetType.Key]; found && metricSetType == core.MetricSetTypePod {
// Aggregating pods
nodeName, found := metricSet.Labels[core.LabelNodename.Key]
if nodeName == "" {
glog.V(8).Infof("Skipping pod %s: no node info", key)
continue
}
if found {
nodeKey := core.NodeKey(nodeName)
node, found := batch.MetricSets[nodeKey]
if !found {
glog.Warningf("Failed to find node: %s", nodeKey)
} else if err := aggregate(metricSet, node, this.MetricsToAggregate); err != nil {
return nil, err
}
} else {
glog.Errorf("No node info in pod %s: %v", key, metricSet.Labels)
}
}
}
return batch, nil
}
// nodeMetrics returns a metric timeseries for a metric of the Node entity.
func (a *Api) nodeMetrics(request *restful.Request, response *restful.Response) {
a.processMetricRequest(core.NodeKey(request.PathParameter("node-name")),
request, response)
}
func TestDecodeSummaryMetrics(t *testing.T) {
ms := testingSummaryMetricsSource()
summary := stats.Summary{
Node: stats.NodeStats{
NodeName: nodeInfo.NodeName,
StartTime: unversioned.NewTime(startTime),
CPU: genTestSummaryCPU(seedNode),
Memory: genTestSummaryMemory(seedNode),
Network: genTestSummaryNetwork(seedNode),
SystemContainers: []stats.ContainerStats{
genTestSummaryContainer(stats.SystemContainerKubelet, seedKubelet),
genTestSummaryContainer(stats.SystemContainerRuntime, seedRuntime),
genTestSummaryContainer(stats.SystemContainerMisc, seedMisc),
},
Fs: genTestSummaryFsStats(seedNode),
},
Pods: []stats.PodStats{{
PodRef: stats.PodReference{
Name: pName0,
Namespace: namespace0,
},
StartTime: unversioned.NewTime(startTime),
Network: genTestSummaryNetwork(seedPod0),
Containers: []stats.ContainerStats{
genTestSummaryContainer(cName00, seedPod0Container0),
genTestSummaryContainer(cName01, seedPod0Container1),
},
}, {
PodRef: stats.PodReference{
Name: pName1,
Namespace: namespace0,
},
StartTime: unversioned.NewTime(startTime),
Network: genTestSummaryNetwork(seedPod1),
Containers: []stats.ContainerStats{
genTestSummaryContainer(cName10, seedPod1Container),
},
VolumeStats: []stats.VolumeStats{{
Name: "A",
FsStats: *genTestSummaryFsStats(seedPod1),
}, {
Name: "B",
FsStats: *genTestSummaryFsStats(seedPod1),
}},
}, {
PodRef: stats.PodReference{
Name: pName2,
Namespace: namespace1,
},
StartTime: unversioned.NewTime(startTime),
Network: genTestSummaryNetwork(seedPod2),
Containers: []stats.ContainerStats{
genTestSummaryContainer(cName20, seedPod2Container),
},
}},
}
containerFs := []string{"/", "logs"}
expectations := []struct {
key string
setType string
seed int64
cpu bool
memory bool
network bool
fs []string
}{{
key: core.NodeKey(nodeInfo.NodeName),
setType: core.MetricSetTypeNode,
seed: seedNode,
cpu: true,
memory: true,
network: true,
fs: []string{"/"},
}, {
key: core.NodeContainerKey(nodeInfo.NodeName, "kubelet"),
setType: core.MetricSetTypeSystemContainer,
seed: seedKubelet,
cpu: true,
memory: true,
}, {
key: core.NodeContainerKey(nodeInfo.NodeName, "docker-daemon"),
setType: core.MetricSetTypeSystemContainer,
seed: seedRuntime,
cpu: true,
memory: true,
}, {
key: core.NodeContainerKey(nodeInfo.NodeName, "system"),
setType: core.MetricSetTypeSystemContainer,
seed: seedMisc,
cpu: true,
memory: true,
}, {
key: core.PodKey(namespace0, pName0),
setType: core.MetricSetTypePod,
seed: seedPod0,
network: true,
}, {
key: core.PodKey(namespace0, pName1),
setType: core.MetricSetTypePod,
seed: seedPod1,
network: true,
fs: []string{"Volume:A", "Volume:B"},
}, {
key: core.PodKey(namespace1, pName2),
setType: core.MetricSetTypePod,
seed: seedPod2,
network: true,
}, {
key: core.PodContainerKey(namespace0, pName0, cName00),
setType: core.MetricSetTypePodContainer,
seed: seedPod0Container0,
cpu: true,
memory: true,
fs: containerFs,
}, {
key: core.PodContainerKey(namespace0, pName0, cName01),
setType: core.MetricSetTypePodContainer,
seed: seedPod0Container1,
cpu: true,
memory: true,
fs: containerFs,
}, {
key: core.PodContainerKey(namespace0, pName1, cName10),
setType: core.MetricSetTypePodContainer,
seed: seedPod1Container,
cpu: true,
memory: true,
fs: containerFs,
}, {
key: core.PodContainerKey(namespace1, pName2, cName20),
setType: core.MetricSetTypePodContainer,
seed: seedPod2Container,
cpu: true,
memory: true,
fs: containerFs,
}}
metrics := ms.decodeSummary(&summary)
for _, e := range expectations {
m, ok := metrics[e.key]
if !assert.True(t, ok, "missing metric %q", e.key) {
continue
}
assert.Equal(t, m.Labels[core.LabelMetricSetType.Key], e.setType, e.key)
assert.Equal(t, m.CreateTime, startTime, e.key)
assert.Equal(t, m.ScrapeTime, scrapeTime, e.key)
if e.cpu {
checkIntMetric(t, m, e.key, core.MetricCpuUsage, e.seed+offsetCPUUsageCoreSeconds)
}
if e.memory {
checkIntMetric(t, m, e.key, core.MetricMemoryUsage, e.seed+offsetMemUsageBytes)
checkIntMetric(t, m, e.key, core.MetricMemoryWorkingSet, e.seed+offsetMemWorkingSetBytes)
checkIntMetric(t, m, e.key, core.MetricMemoryPageFaults, e.seed+offsetMemPageFaults)
checkIntMetric(t, m, e.key, core.MetricMemoryMajorPageFaults, e.seed+offsetMemMajorPageFaults)
}
if e.network {
checkIntMetric(t, m, e.key, core.MetricNetworkRx, e.seed+offsetNetRxBytes)
checkIntMetric(t, m, e.key, core.MetricNetworkRxErrors, e.seed+offsetNetRxErrors)
checkIntMetric(t, m, e.key, core.MetricNetworkTx, e.seed+offsetNetTxBytes)
checkIntMetric(t, m, e.key, core.MetricNetworkTxErrors, e.seed+offsetNetTxErrors)
}
for _, label := range e.fs {
checkFsMetric(t, m, e.key, label, core.MetricFilesystemAvailable, e.seed+offsetFsAvailable)
checkFsMetric(t, m, e.key, label, core.MetricFilesystemLimit, e.seed+offsetFsCapacity)
checkFsMetric(t, m, e.key, label, core.MetricFilesystemUsage, e.seed+offsetFsUsed)
}
delete(metrics, e.key)
}
for k, v := range metrics {
assert.Fail(t, "unexpected metric", "%q: %+v", k, v)
}
}
func TestNodeAggregate(t *testing.T) {
batch := core.DataBatch{
Timestamp: time.Now(),
MetricSets: map[string]*core.MetricSet{
core.PodKey("ns1", "pod1"): {
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePod,
core.LabelNamespaceName.Key: "ns1",
core.LabelNodename.Key: "h1",
},
MetricValues: map[string]core.MetricValue{
"m1": {
ValueType: core.ValueInt64,
MetricType: core.MetricGauge,
IntValue: 10,
},
"m2": {
ValueType: core.ValueInt64,
MetricType: core.MetricGauge,
IntValue: 222,
},
},
},
core.PodKey("ns1", "pod2"): {
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypePod,
core.LabelNamespaceName.Key: "ns1",
core.LabelNodename.Key: "h1",
},
MetricValues: map[string]core.MetricValue{
"m1": {
ValueType: core.ValueInt64,
MetricType: core.MetricGauge,
IntValue: 100,
},
"m3": {
ValueType: core.ValueInt64,
MetricType: core.MetricGauge,
IntValue: 30,
},
},
},
core.NodeKey("h1"): {
Labels: map[string]string{
core.LabelMetricSetType.Key: core.MetricSetTypeNode,
core.LabelNodename.Key: "h1",
},
MetricValues: map[string]core.MetricValue{},
},
},
}
processor := NodeAggregator{
MetricsToAggregate: []string{"m1", "m3"},
}
result, err := processor.Process(&batch)
assert.NoError(t, err)
node, found := result.MetricSets[core.NodeKey("h1")]
assert.True(t, found)
m1, found := node.MetricValues["m1"]
assert.True(t, found)
assert.Equal(t, 110, m1.IntValue)
m3, found := node.MetricValues["m3"]
assert.True(t, found)
assert.Equal(t, 30, m3.IntValue)
}