// Recursively traverse the Executor struct, building "/"-delimited strings that resemble snap metric types. If a given
// feature is not enabled on a Mesos agent (e.g. the network isolator), then those metrics will be removed from the
// metric types returned by this function.
func GetMonitoringStatisticsMetricTypes(host string) ([]string, error) {
log.Debug("Getting monitoring statistics metrics type from host ", host)
// TODO(roger): supporting NetTrafficControlStatistics means adding another dynamic metric to the plugin.
// When we're ready to do this, remove ns.InspectEmptyContainers(ns.AlwaysFalse) so this defaults to true.
namespaces := []string{}
err := ns.FromCompositeObject(
&mesos_pb2.ResourceStatistics{}, "", &namespaces, ns.InspectEmptyContainers(ns.AlwaysFalse))
if err != nil {
log.Error(err)
return nil, err
}
// Avoid returning a metric type that is impossible to collect on this system
flags, err := GetFlags(host)
if err != nil {
log.Error(err)
return nil, err
}
// Isolators are defined using a comma-separated string passed to the "--isolation" flag on the Mesos agent.
// See https://github.com/apache/mesos/blob/0.28.1/src/slave/containerizer/mesos/containerizer.cpp#L196-L223
isolators := strings.Split(flags["isolation"], ",")
if str.Contains(isolators, "cgroups/perf_event") {
log.Debug("Isolator cgroups/perf_event is enabled on host ", host)
// Expects a perf event from the output of `perf list`. Mesos then normalizes the event name. See
// https://github.com/apache/mesos/blob/0.28.1/src/linux/perf.cpp#L65-L71
namespaces = deleteFromSlice(namespaces, "^perf/.*")
var normalizedPerfEvents []string
perfEvents := strings.Split(flags["perf_events"], ",")
for _, event := range perfEvents {
log.Debug("Adding perf event ", event, " to metrics catalog")
event = fmt.Sprintf("perf/%s", normalizePerfEventName(event))
normalizedPerfEvents = append(normalizedPerfEvents, event)
}
namespaces = append(namespaces, normalizedPerfEvents...)
} else {
log.Debug("Isolator cgroups/perf_event is not enabled on host ", host)
namespaces = deleteFromSlice(namespaces, "^perf.*")
}
if !str.Contains(isolators, "posix/disk") {
log.Debug("Isolator posix/disk is not enabled on host ", host)
namespaces = deleteFromSlice(namespaces, "^disk_.*")
}
if !str.Contains(isolators, "network/port_mapping") {
log.Debug("Isolator network/port_mapping is not enabled on host ", host)
namespaces = deleteFromSlice(namespaces, "^net_.*")
}
return namespaces, nil
}
func TestGetMetricTypes(t *testing.T) {
Convey("Tesing GetMetricTypes", t, func() {
newPlg := new(Mock)
So(newPlg, ShouldNotBeNil)
Convey("with missing on-load plugin config entry", func() {
cfg := plugin.Config{"test-fail": true}
_, err := newPlg.GetMetricTypes(cfg)
So(err, ShouldNotBeNil)
})
Convey("with 'test' config variable", func() {
cfg := plugin.Config{"test": true}
mts, err := newPlg.GetMetricTypes(cfg)
So(err, ShouldBeNil)
So(len(mts), ShouldEqual, 4)
Convey("checking namespaces", func() {
metricNames := []string{}
for _, m := range mts {
metricNames = append(metricNames, m.Namespace.String())
}
ns := plugin.NewNamespace("intel", "mock", "test")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = plugin.NewNamespace("intel", "mock", "foo")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = plugin.NewNamespace("intel", "mock", "bar")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = plugin.NewNamespace("intel", "mock").AddDynamicElement("host", "name of the host").AddStaticElement("baz")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
})
})
Convey("without config variables", func() {
mts, err := newPlg.GetMetricTypes(plugin.Config{})
So(err, ShouldBeNil)
So(len(mts), ShouldEqual, 3)
Convey("checking namespaces", func() {
metricNames := []string{}
for _, m := range mts {
metricNames = append(metricNames, m.Namespace.String())
}
ns := plugin.NewNamespace("intel", "mock", "foo")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = plugin.NewNamespace("intel", "mock", "bar")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = plugin.NewNamespace("intel", "mock").AddDynamicElement("host", "name of the host").AddStaticElement("baz")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
})
})
})
}
func TestGetMetricTypes(t *testing.T) {
Convey("Tesing GetMetricTypes", t, func() {
newPlg := new(AnotherMock)
So(newPlg, ShouldNotBeNil)
Convey("with missing on-load plugin config entry", func() {
node := cdata.NewNode()
node.AddItem("test-fail", ctypes.ConfigValueStr{Value: ""})
_, err := newPlg.GetMetricTypes(plugin.ConfigType{ConfigDataNode: node})
So(err, ShouldNotBeNil)
})
Convey("with 'test' config variable", func() {
node := cdata.NewNode()
node.AddItem("test", ctypes.ConfigValueStr{Value: ""})
mts, err := newPlg.GetMetricTypes(plugin.ConfigType{ConfigDataNode: node})
So(err, ShouldBeNil)
So(len(mts), ShouldEqual, 4)
Convey("checking namespaces", func() {
metricNames := []string{}
for _, m := range mts {
metricNames = append(metricNames, m.Namespace().String())
}
ns := core.NewNamespace("intel", "anothermock", "test")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = core.NewNamespace("intel", "anothermock", "foo")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = core.NewNamespace("intel", "anothermock", "bar")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = core.NewNamespace("intel", "anothermock").AddDynamicElement("host", "name of the host").AddStaticElement("baz")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
})
})
Convey("without config variables", func() {
node := cdata.NewNode()
mts, err := newPlg.GetMetricTypes(plugin.ConfigType{ConfigDataNode: node})
So(err, ShouldBeNil)
So(len(mts), ShouldEqual, 3)
Convey("checking namespaces", func() {
metricNames := []string{}
for _, m := range mts {
metricNames = append(metricNames, m.Namespace().String())
}
ns := core.NewNamespace("intel", "anothermock", "foo")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = core.NewNamespace("intel", "anothermock", "bar")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
ns = core.NewNamespace("intel", "anothermock").AddDynamicElement("host", "name of the host").AddStaticElement("baz")
So(str.Contains(metricNames, ns.String()), ShouldBeTrue)
})
})
})
}
func (s *CollectorSuite) TestGetMetricTypes() {
Convey("Given config with enpoint, user and password defined", s.T(), func() {
cfg := setupCfg(s.server.URL, "me", "secret", "admin")
Convey("When GetMetricTypes() is called", func() {
collector := New()
mts, err := collector.GetMetricTypes(cfg)
Convey("Then no error should be reported", func() {
So(err, ShouldBeNil)
})
Convey("and proper metric types are returned", func() {
metricNames := []string{}
for _, m := range mts {
metricNames = append(metricNames, m.Namespace().String())
}
So(len(mts), ShouldEqual, 12)
So(str.Contains(metricNames, "/intel/openstack/cinder/demo/snapshots/count"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/demo/snapshots/bytes"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/demo/volumes/count"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/demo/volumes/bytes"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/demo/limits/MaxTotalVolumeGigabytes"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/demo/limits/MaxTotalVolumes"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/admin/volumes/count"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/admin/volumes/bytes"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/admin/limits/MaxTotalVolumeGigabytes"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/admin/limits/MaxTotalVolumes"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/admin/snapshots/count"), ShouldBeTrue)
So(str.Contains(metricNames, "/intel/openstack/cinder/admin/snapshots/bytes"), ShouldBeTrue)
})
})
})
}
// CollectMetrics returns list of requested metric values
// It returns error in case retrieval was not successful
func (c *collector) CollectMetrics(metricTypes []plugin.MetricType) ([]plugin.MetricType, error) {
// get admin tenant from configuration. admin tenant is needed for gathering volumes and snapshots metrics at once
item, err := config.GetConfigItem(metricTypes[0], "tenant")
if err != nil {
return nil, err
}
admin := item.(string)
// populate information about all available tenants
if len(c.allTenants) == 0 {
c.allTenants, err = getTenants(metricTypes[0])
if err != nil {
return nil, err
}
}
// iterate over metric types to resolve needed collection calls
// for requested tenants
collectTenants := str.InitSet()
var collectLimits, collectVolumes, collectSnapshots bool
for _, metricType := range metricTypes {
namespace := metricType.Namespace()
if len(namespace) < 6 {
return nil, fmt.Errorf("Incorrect namespace lenth. Expected 6 is %d", len(namespace))
}
tenant := namespace[3].Value
collectTenants.Add(tenant)
if str.Contains(namespace.Strings(), "limits") {
collectLimits = true
} else if str.Contains(namespace.Strings(), "volumes") {
collectVolumes = true
} else {
collectSnapshots = true
}
}
allSnapshots := map[string]types.Snapshots{}
allVolumes := map[string]types.Volumes{}
// collect volumes and snapshots separately by authenticating to admin
{
if err := c.authenticate(metricTypes[0], admin); err != nil {
return nil, err
}
provider := c.providers[admin]
var done sync.WaitGroup
errChn := make(chan error, 2)
// Collect volumes
if collectVolumes {
done.Add(1)
go func() {
defer done.Done()
volumes, err := c.service.GetVolumes(provider)
if err != nil {
errChn <- err
}
for tenantId, volumeCount := range volumes {
tenantName := c.allTenants[tenantId]
allVolumes[tenantName] = volumeCount
}
}()
}
// Collect snapshots
if collectSnapshots {
done.Add(1)
go func() {
defer done.Done()
snapshots, err := c.service.GetSnapshots(provider)
if err != nil {
errChn <- err
}
for tenantId, snapshotCount := range snapshots {
tenantName := c.allTenants[tenantId]
allSnapshots[tenantName] = snapshotCount
}
}()
}
done.Wait()
close(errChn)
if e := <-errChn; e != nil {
return nil, e
}
}
// Collect limits per each tenant only if not already collected (plugin lifetime scope)
{
var done sync.WaitGroup
errChn := make(chan error, collectTenants.Size())
for _, tenant := range collectTenants.Elements() {
_, found := c.allLimits[tenant]
if collectLimits && !found {
if err := c.authenticate(metricTypes[0], tenant); err != nil {
return nil, err
}
provider := c.providers[tenant]
done.Add(1)
go func(p *gophercloud.ProviderClient, t string) {
defer done.Done()
limits, err := c.service.GetLimits(p)
if err != nil {
errChn <- err
}
c.allLimits[t] = limits
}(provider, tenant)
}
}
done.Wait()
close(errChn)
if e := <-errChn; e != nil {
return nil, e
}
}
metrics := []plugin.MetricType{}
for _, metricType := range metricTypes {
namespace := metricType.Namespace().Strings()
tenant := namespace[3]
// Construct temporary struct to accommodate all gathered metrics
metricContainer := struct {
S types.Snapshots `json:"snapshots"`
V types.Volumes `json:"volumes"`
L types.Limits `json:"limits"`
}{
allSnapshots[tenant],
allVolumes[tenant],
c.allLimits[tenant],
}
// Extract values by namespace from temporary struct and create metrics
metric := plugin.MetricType{
Timestamp_: time.Now(),
Namespace_: metricType.Namespace(),
Data_: ns.GetValueByNamespace(metricContainer, namespace[4:]),
}
metrics = append(metrics, metric)
}
return metrics, nil
}
