func TestApiFactory(t *testing.T) {
metricSink := metricsink.MetricSink{}
api := NewApi(false, &metricSink, nil)
as := assert.New(t)
for _, metric := range core.StandardMetrics {
val, exists := api.gkeMetrics[metric.Name]
as.True(exists)
as.Equal(val, metric.MetricDescriptor)
}
for _, metric := range core.LabeledMetrics {
val, exists := api.gkeMetrics[metric.Name]
as.True(exists)
as.Equal(val, metric.MetricDescriptor)
}
for _, metric := range core.LabeledMetrics {
val, exists := api.gkeMetrics[metric.Name]
as.True(exists)
as.Equal(val, metric.MetricDescriptor)
}
labels := append(core.CommonLabels(), core.ContainerLabels()...)
labels = append(labels, core.PodLabels()...)
for _, label := range labels {
val, exists := api.gkeLabels[label.Key]
as.True(exists)
as.Equal(val, label)
}
}
// Create a new Api to serve from the specified cache.
func NewApi(runningInKubernetes bool, metricSink *metricsink.MetricSink, historicalSource core.HistoricalSource) *Api {
gkeMetrics := make(map[string]core.MetricDescriptor)
gkeLabels := make(map[string]core.LabelDescriptor)
for _, val := range core.StandardMetrics {
gkeMetrics[val.Name] = val.MetricDescriptor
}
for _, val := range core.LabeledMetrics {
gkeMetrics[val.Name] = val.MetricDescriptor
}
gkeMetrics[core.MetricCpuLimit.Name] = core.MetricCpuLimit.MetricDescriptor
gkeMetrics[core.MetricMemoryLimit.Name] = core.MetricMemoryLimit.MetricDescriptor
for _, val := range core.CommonLabels() {
gkeLabels[val.Key] = val
}
for _, val := range core.ContainerLabels() {
gkeLabels[val.Key] = val
}
for _, val := range core.PodLabels() {
gkeLabels[val.Key] = val
}
return &Api{
runningInKubernetes: runningInKubernetes,
metricSink: metricSink,
historicalSource: historicalSource,
gkeMetrics: gkeMetrics,
gkeLabels: gkeLabels,
}
}
func (a *Api) exportMetricsSchema(_ *restful.Request, response *restful.Response) {
result := types.TimeseriesSchema{
Metrics: make([]types.MetricDescriptor, 0),
CommonLabels: make([]types.LabelDescriptor, 0),
PodLabels: make([]types.LabelDescriptor, 0),
}
for _, metric := range core.StandardMetrics {
if _, found := a.gkeMetrics[metric.Name]; found {
result.Metrics = append(result.Metrics, convertMetricDescriptor(metric.MetricDescriptor))
}
}
for _, metric := range core.AdditionalMetrics {
if _, found := a.gkeMetrics[metric.Name]; found {
result.Metrics = append(result.Metrics, convertMetricDescriptor(metric.MetricDescriptor))
}
}
for _, metric := range core.LabeledMetrics {
if _, found := a.gkeMetrics[metric.Name]; found {
result.Metrics = append(result.Metrics, convertMetricDescriptor(metric.MetricDescriptor))
}
}
for _, label := range core.CommonLabels() {
if _, found := a.gkeLabels[label.Key]; found {
result.CommonLabels = append(result.CommonLabels, convertLabelDescriptor(label))
}
}
for _, label := range core.ContainerLabels() {
if _, found := a.gkeLabels[label.Key]; found {
result.CommonLabels = append(result.CommonLabels, convertLabelDescriptor(label))
}
}
for _, label := range core.PodLabels() {
if _, found := a.gkeLabels[label.Key]; found {
result.PodLabels = append(result.PodLabels, convertLabelDescriptor(label))
}
}
response.WriteEntity(result)
}
func TestRealInput(t *testing.T) {
api := NewApi(false, nil, nil)
dataBatch := []*core.DataBatch{
{
Timestamp: time.Now(),
MetricSets: map[string]*core.MetricSet{},
},
{
Timestamp: time.Now().Add(-time.Minute),
MetricSets: map[string]*core.MetricSet{},
},
}
labels := append(core.CommonLabels(), core.ContainerLabels()...)
labels = append(labels, core.PodLabels()...)
for _, entry := range dataBatch {
// Add a pod, container, node, systemcontainer
entry.MetricSets[core.MetricSetTypePod] = generateMetricSet(core.MetricSetTypePod, labels)
entry.MetricSets[core.MetricSetTypeNode] = generateMetricSet(core.MetricSetTypeNode, labels)
entry.MetricSets[core.MetricSetTypePodContainer] = generateMetricSet(core.MetricSetTypePodContainer, labels)
entry.MetricSets[core.MetricSetTypeSystemContainer] = generateMetricSet(core.MetricSetTypeSystemContainer, labels)
}
ts := api.processMetricsRequest(dataBatch)
type expectation struct {
count int
extraLabels bool
}
expectedMetrics := make(map[string]*expectation)
for _, metric := range core.StandardMetrics {
expectedMetrics[metric.Name] = &expectation{
count: 4,
extraLabels: false,
}
}
for _, metric := range core.LabeledMetrics {
expectedMetrics[metric.Name] = &expectation{
count: 4,
extraLabels: true,
}
}
as := assert.New(t)
for _, elem := range ts {
// validate labels
for _, label := range labels {
val, exists := elem.Labels[label.Key]
as.True(exists, "%q label does not exist", label.Key)
if label.Key == core.LabelMetricSetType.Key {
continue
}
if label.Key == core.LabelContainerName.Key && val != "machine" && val != "/pod" {
as.Equal(val, "test-value", "%q label's value is %q, expected 'test-value'", label.Key, val)
}
}
for mname, points := range elem.Metrics {
ex := expectedMetrics[mname]
require.NotNil(t, ex)
as.NotEqual(ex, 0)
ex.count--
for _, point := range points {
as.Equal(point.Value, -1)
if !ex.extraLabels {
continue
}
as.Equal(len(core.MetricLabels()), len(point.Labels))
for _, label := range core.MetricLabels() {
val, exists := point.Labels[label.Key]
as.True(exists, "expected label %q to be found - %+v", label.Key, point.Labels)
as.Equal(val, "test-value")
}
}
}
}
}
func runMetricExportTest(fm kubeFramework, svc *kube_api.Service) error {
podList, err := fm.GetPodsRunningOnNodes()
if err != nil {
return err
}
expectedPods := make([]string, 0, len(podList))
for _, pod := range podList {
expectedPods = append(expectedPods, pod.Name)
}
glog.V(0).Infof("Expected pods: %v", expectedPods)
expectedNodes, err := fm.GetNodeNames()
if err != nil {
return err
}
glog.V(0).Infof("Expected nodes: %v", expectedNodes)
timeseries, err := getTimeseries(fm, svc)
if err != nil {
return err
}
if len(timeseries) == 0 {
return fmt.Errorf("expected non zero timeseries")
}
schema, err := getSchema(fm, svc)
if err != nil {
return err
}
// Build a map of metric names to metric descriptors.
mdMap := map[string]*api_v1.MetricDescriptor{}
for idx := range schema.Metrics {
mdMap[schema.Metrics[idx].Name] = &schema.Metrics[idx]
}
actualPods := map[string]bool{}
actualNodes := map[string]bool{}
actualSystemContainers := map[string]map[string]struct{}{}
for _, ts := range timeseries {
// Verify the relevant labels are present.
// All common labels must be present.
podName, podMetric := ts.Labels[core.LabelPodName.Key]
for _, label := range core.CommonLabels() {
_, exists := ts.Labels[label.Key]
if !exists {
return fmt.Errorf("timeseries: %v does not contain common label: %v", ts, label)
}
}
if podMetric {
for _, label := range core.PodLabels() {
_, exists := ts.Labels[label.Key]
if !exists {
return fmt.Errorf("timeseries: %v does not contain pod label: %v", ts, label)
}
}
}
if podMetric {
actualPods[podName] = true
// Extra explicit check that the expecte metrics are there:
requiredLabels := []string{
core.LabelPodNamespaceUID.Key,
core.LabelPodId.Key,
core.LabelHostID.Key,
// container name is checked later
}
for _, label := range requiredLabels {
_, exists := ts.Labels[label]
if !exists {
return fmt.Errorf("timeseries: %v does not contain required label: %v", ts, label)
}
}
} else {
if cName, ok := ts.Labels[core.LabelContainerName.Key]; ok {
hostname, ok := ts.Labels[core.LabelHostname.Key]
if !ok {
return fmt.Errorf("hostname label missing on container %+v", ts)
}
if cName == "machine" {
actualNodes[hostname] = true
} else {
for _, label := range core.ContainerLabels() {
if label == core.LabelContainerBaseImage && !isContainerBaseImageExpected(ts) {
continue
}
_, exists := ts.Labels[label.Key]
if !exists {
return fmt.Errorf("timeseries: %v does not contain container label: %v", ts, label)
}
}
}
if _, exists := expectedSystemContainers[cName]; exists {
if actualSystemContainers[cName] == nil {
actualSystemContainers[cName] = map[string]struct{}{}
}
actualSystemContainers[cName][hostname] = struct{}{}
}
} else {
return fmt.Errorf("container_name label missing on timeseries - %v", ts)
}
}
// Explicitly check for resource id
explicitRequirement := map[string][]string{
core.MetricFilesystemUsage.MetricDescriptor.Name: {core.LabelResourceID.Key},
core.MetricFilesystemLimit.MetricDescriptor.Name: {core.LabelResourceID.Key},
core.MetricFilesystemAvailable.Name: {core.LabelResourceID.Key}}
for metricName, points := range ts.Metrics {
md, exists := mdMap[metricName]
if !exists {
return fmt.Errorf("unexpected metric %q", metricName)
}
for _, point := range points {
for _, label := range md.Labels {
_, exists := point.Labels[label.Key]
if !exists {
return fmt.Errorf("metric %q point %v does not contain metric label: %v", metricName, point, label)
}
}
}
required := explicitRequirement[metricName]
for _, label := range required {
for _, point := range points {
_, exists := point.Labels[label]
if !exists {
return fmt.Errorf("metric %q point %v does not contain metric label: %v", metricName, point, label)
}
}
}
}
}
// Validate that system containers are running on all the nodes.
// This test could fail if one of the containers was down while the metrics sample was collected.
for cName, hosts := range actualSystemContainers {
for _, host := range expectedNodes {
if _, ok := hosts[host]; !ok {
return fmt.Errorf("System container %q not found on host: %q - %v", cName, host, actualSystemContainers)
}
}
}
if err := expectedItemsExist(expectedPods, actualPods); err != nil {
return fmt.Errorf("expected pods don't exist %v.\nExpected: %v\nActual:%v", err, expectedPods, actualPods)
}
if err := expectedItemsExist(expectedNodes, actualNodes); err != nil {
return fmt.Errorf("expected nodes don't exist %v.\nExpected: %v\nActual:%v", err, expectedNodes, actualNodes)
}
return nil
}