// createMetricsSubArray creates an array of metrics for map values returned by facter
func createMetricsSubArray(name string, value map[string]interface{}) ([]plugin.MetricType, error) {
var ms []plugin.MetricType
namespaces := []string{}
err := utils.FromMap(value, name, &namespaces)
if err != nil {
return nil, err
}
for _, namespace := range namespaces {
ns := strings.Split(namespace, "/")
val := utils.GetValueByNamespace(value, ns[1:])
m := *plugin.NewMetricType(createNamespace(ns...), time.Now(), nil, "", val)
ms = append(ms, m)
}
return ms, nil
}
func (m *Mesos) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) {
configItems, err := getConfig(mts[0])
if err != nil {
return nil, err
}
requestedMaster := []core.Namespace{}
requestedAgent := []core.Namespace{}
for _, metricType := range mts {
switch metricType.Namespace().Strings()[2] {
case "master":
requestedMaster = append(requestedMaster, metricType.Namespace())
case "agent":
requestedAgent = append(requestedAgent, metricType.Namespace())
}
}
// Translate Mesos metrics into Snap PluginMetrics
now := time.Now()
metrics := []plugin.MetricType{}
if configItems["master"] != "" && len(requestedMaster) > 0 {
log.Info("Collecting ", len(requestedMaster), " metrics from the master")
isLeader, err := master.IsLeader(configItems["master"])
if err != nil {
log.Error(err)
return nil, err
}
if isLeader {
snapshot, err := master.GetMetricsSnapshot(configItems["master"])
if err != nil {
log.Error(err)
return nil, err
}
frameworks, err := master.GetFrameworks(configItems["master"])
if err != nil {
log.Error(err)
return nil, err
}
tags := map[string]string{"source": configItems["master"]}
for _, requested := range requestedMaster {
isDynamic, _ := requested.IsDynamic()
if isDynamic {
n := requested.Strings()[4:]
// Iterate through the array of frameworks returned by GetFrameworks()
for _, framework := range frameworks {
val := ns.GetValueByNamespace(framework, n)
if val == nil {
log.Warn("Attempted to collect metric ", requested.String(), " but it returned nil!")
continue
}
// substituting "framework" wildcard with particular framework id
requested[3].Value = framework.ID
// TODO(roger): units
metrics = append(metrics, *plugin.NewMetricType(requested, now, tags, "", val))
}
} else {
n := requested.Strings()[3:]
val, ok := snapshot[strings.Join(n, "/")]
if !ok {
e := fmt.Errorf("error: requested metric %s not found", requested.String())
log.Error(e)
return nil, e
}
//TODO(kromar): is it possible to provide unit NewMetricType(ns, time, tags, unit, value)?
// I'm leaving empty string for now...
metrics = append(metrics, *plugin.NewMetricType(requested, now, tags, "", val))
}
}
} else {
log.Info("Attempted CollectMetrics() on ", configItems["master"], "but it isn't the leader. Skipping...")
}
}
if configItems["agent"] != "" && len(requestedAgent) > 0 {
log.Info("Collecting ", len(requestedAgent), " metrics from the agent")
snapshot, err := agent.GetMetricsSnapshot(configItems["agent"])
if err != nil {
log.Error(err)
return nil, err
}
executors, err := agent.GetMonitoringStatistics(configItems["agent"])
if err != nil {
log.Error(err)
return nil, err
}
tags := map[string]string{"source": configItems["agent"]}
for _, requested := range requestedAgent {
n := requested.Strings()[5:]
isDynamic, _ := requested.IsDynamic()
if isDynamic {
// Iterate through the array of executors returned by GetMonitoringStatistics()
for _, exec := range executors {
val := ns.GetValueByNamespace(exec.Statistics, n)
if val == nil {
log.Warn("Attempted to collect metric ", requested.String(), " but it returned nil!")
continue
}
// substituting "framework" wildcard with particular framework id
requested[3].Value = exec.Framework
// substituting "executor" wildcard with particular executor id
requested[4].Value = exec.ID
// TODO(roger): units
metrics = append(metrics, *plugin.NewMetricType(requested, now, tags, "", val))
}
} else {
// Get requested metrics from the snapshot map
n := requested.Strings()[3:]
val, ok := snapshot[strings.Join(n, "/")]
if !ok {
e := fmt.Errorf("error: requested metric %v not found", requested.String())
log.Error(e)
return nil, e
}
//TODO(kromar): units here also?
metrics = append(metrics, *plugin.NewMetricType(requested, now, tags, "", val))
}
}
}
log.Debug("Collected a total of ", len(metrics), " metrics.")
return metrics, nil
}
// CollectMetrics retrieves values of requested metrics
func (d *docker) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) {
var err error
metrics := []plugin.MetricType{}
d.list = map[string]dock.APIContainers{}
// get list of possible network metrics
networkMetrics := []string{}
utils.FromCompositionTags(wrapper.NetworkInterface{}, "", &networkMetrics)
// get list of all running containers
d.list, err = d.client.ListContainersAsMap()
if err != nil {
fmt.Fprintln(os.Stderr, "The list of running containers cannot be retrived, err=", err)
return nil, err
}
// retrieve requested docker ids
rids, err := d.getRequestedIDs(mts...)
if err != nil {
return nil, err
}
// for each requested id set adequate item into docker.container struct with stats
for _, rid := range rids {
if contSpec, exist := d.list[rid]; !exist {
return nil, fmt.Errorf("Docker container does not exist, container_id=%s", rid)
} else {
stats, err := d.client.GetStatsFromContainer(contSpec.ID, true)
if err != nil {
return nil, err
}
// set new item to docker.container structure
d.containers[rid] = containerData{
ID: contSpec.ID,
Info: wrapper.Specification{
Status: contSpec.Status,
Created: time.Unix(contSpec.Created, 0).Format("2006-01-02T15:04:05Z07:00"),
Image: contSpec.Image,
SizeRw: contSpec.SizeRw,
SizeRootFs: contSpec.SizeRootFs,
Labels: contSpec.Labels,
},
Stats: stats,
}
}
}
for _, mt := range mts {
ids, err := d.getRequestedIDs(mt)
if err != nil {
return nil, err
}
for _, id := range ids {
ns := make([]core.NamespaceElement, len(mt.Namespace()))
copy(ns, mt.Namespace())
ns[2].Value = id
// omit "spec" metrics for root
if id == "root" && mt.Namespace()[lengthOfNsPrefix].Value == "spec" {
continue
}
isDynamic, indexes := mt.Namespace()[lengthOfNsPrefix:].IsDynamic()
metricName := mt.Namespace().Strings()[lengthOfNsPrefix:]
// remove added static element (`value`)
if metricName[len(metricName)-1] == "value" {
metricName = metricName[:len(metricName)-1]
}
if !isDynamic {
metric := plugin.MetricType{
Timestamp_: time.Now(),
Namespace_: ns,
Data_: utils.GetValueByNamespace(d.containers[id], metricName),
Tags_: mt.Tags(),
Config_: mt.Config(),
Version_: VERSION,
}
metrics = append(metrics, metric)
continue
}
// take the element of metricName which precedes the first dynamic element
// e.g. {"filesystem", "*", "usage"}
// -> statsType will be "filesystem",
// -> scope of metricName will be decreased to {"*", "usage"}
indexOfDynamicElement := indexes[0]
statsType := metricName[indexOfDynamicElement-1]
metricName = metricName[indexOfDynamicElement:]
switch statsType {
case "filesystem":
// get docker filesystem statistics
devices := []string{}
if metricName[0] == "*" {
// when device name is requested as as asterisk - take all available filesystem devices
for deviceName := range d.containers[id].Stats.Filesystem {
devices = append(devices, deviceName)
}
} else {
// device name is requested explicitly
device := metricName[0]
if _, ok := d.containers[id].Stats.Filesystem[device]; !ok {
return nil, fmt.Errorf("In metric %s the given device name is invalid (no stats for this device)", mt.Namespace().String())
}
devices = append(devices, metricName[0])
}
for _, device := range devices {
rns := make([]core.NamespaceElement, len(ns))
copy(rns, ns)
rns[indexOfDynamicElement+lengthOfNsPrefix].Value = device
metric := plugin.MetricType{
Timestamp_: time.Now(),
Namespace_: rns,
Data_: utils.GetValueByNamespace(d.containers[id].Stats.Filesystem[device], metricName[1:]),
Tags_: mt.Tags(),
Config_: mt.Config(),
Version_: VERSION,
}
metrics = append(metrics, metric)
}
case "labels":
// get docker labels
labelKeys := []string{}
if metricName[0] == "*" {
// when label key is requested as an asterisk - take all available labels
for labelKey := range d.containers[id].Info.Labels {
labelKeys = append(labelKeys, labelKey)
}
} else {
labelKey := metricName[0]
if _, ok := d.containers[id].Info.Labels[labelKey]; !ok {
return nil, fmt.Errorf("In metric %s the given label is invalid (no value for this label key)", mt.Namespace().String())
}
labelKeys = append(labelKeys, metricName[0])
}
for _, labelKey := range labelKeys {
rns := make([]core.NamespaceElement, len(ns))
copy(rns, ns)
rns[indexOfDynamicElement+lengthOfNsPrefix].Value = utils.ReplaceNotAllowedCharsInNamespacePart(labelKey)
metric := plugin.MetricType{
Timestamp_: time.Now(),
Namespace_: rns,
Data_: d.containers[id].Info.Labels[labelKey],
Tags_: mt.Tags(),
Config_: mt.Config(),
Version_: VERSION,
}
metrics = append(metrics, metric)
}
case "network":
//get docker network tx/rx statistics
netInterfaces := []string{}
ifaceMap := map[string]wrapper.NetworkInterface{}
for _, iface := range d.containers[id].Stats.Network {
ifaceMap[iface.Name] = iface
}
// support wildcard on interface name
if metricName[0] == "*" {
for _, netInterface := range d.containers[id].Stats.Network {
netInterfaces = append(netInterfaces, netInterface.Name)
}
} else {
netInterface := metricName[0]
if _, ok := ifaceMap[netInterface]; !ok {
return nil, fmt.Errorf("In metric %s the given network interface is invalid (no stats for this net interface)", mt.Namespace().String())
}
netInterfaces = append(netInterfaces, metricName[0])
}
for _, ifaceName := range netInterfaces {
rns := make([]core.NamespaceElement, len(ns))
copy(rns, ns)
rns[indexOfDynamicElement+lengthOfNsPrefix].Value = ifaceName
metric := plugin.MetricType{
Timestamp_: time.Now(),
Namespace_: rns,
Data_: utils.GetValueByNamespace(ifaceMap[ifaceName], metricName[1:]),
Tags_: mt.Tags(),
Config_: mt.Config(),
Version_: VERSION,
}
metrics = append(metrics, metric)
}
case "percpu_usage":
numOfCPUs := len(d.containers[id].Stats.CgroupStats.CpuStats.CpuUsage.PercpuUsage) - 1
if metricName[0] == "*" {
// when cpu ID is requested as an asterisk - take all available
for cpuID, val := range d.containers[id].Stats.CgroupStats.CpuStats.CpuUsage.PercpuUsage {
rns := make([]core.NamespaceElement, len(ns))
copy(rns, ns)
rns[indexOfDynamicElement+lengthOfNsPrefix].Value = strconv.Itoa(cpuID)
metric := plugin.MetricType{
Timestamp_: time.Now(),
Namespace_: rns,
Data_: val,
Tags_: mt.Tags(),
Config_: mt.Config(),
Version_: VERSION,
}
metrics = append(metrics, metric)
}
} else {
cpuID, err := strconv.Atoi(metricName[0])
if err != nil {
return nil, fmt.Errorf("In metric %s the given cpu id is invalid, err=%v", mt.Namespace().String(), err)
}
if cpuID > numOfCPUs || cpuID < 0 {
return nil, fmt.Errorf("In metric %s the given cpu id is invalid, expected value in range 0-%d", mt.Namespace().String(), numOfCPUs)
}
metric := plugin.MetricType{
Timestamp_: time.Now(),
Namespace_: ns,
Data_: d.containers[id].Stats.CgroupStats.CpuStats.CpuUsage.PercpuUsage[cpuID],
Tags_: mt.Tags(),
Config_: mt.Config(),
Version_: VERSION,
}
metrics = append(metrics, metric)
}
} // the end of switch statsType
} // the end of range over ids
}
if len(metrics) == 0 {
return nil, errors.New("No metric found")
}
return metrics, nil
}
// CollectMetrics returns list of requested metric values
// It returns error in case retrieval was not successful
func (c *collector) CollectMetrics(metricTypes []plugin.MetricType) ([]plugin.MetricType, error) {
// get admin tenant from configuration. admin tenant is needed for gathering volumes and snapshots metrics at once
item, err := config.GetConfigItem(metricTypes[0], "tenant")
if err != nil {
return nil, err
}
admin := item.(string)
// populate information about all available tenants
if len(c.allTenants) == 0 {
c.allTenants, err = getTenants(metricTypes[0])
if err != nil {
return nil, err
}
}
// iterate over metric types to resolve needed collection calls
// for requested tenants
collectTenants := str.InitSet()
var collectLimits, collectVolumes, collectSnapshots bool
for _, metricType := range metricTypes {
namespace := metricType.Namespace()
if len(namespace) < 6 {
return nil, fmt.Errorf("Incorrect namespace lenth. Expected 6 is %d", len(namespace))
}
tenant := namespace[3].Value
collectTenants.Add(tenant)
if str.Contains(namespace.Strings(), "limits") {
collectLimits = true
} else if str.Contains(namespace.Strings(), "volumes") {
collectVolumes = true
} else {
collectSnapshots = true
}
}
allSnapshots := map[string]types.Snapshots{}
allVolumes := map[string]types.Volumes{}
// collect volumes and snapshots separately by authenticating to admin
{
if err := c.authenticate(metricTypes[0], admin); err != nil {
return nil, err
}
provider := c.providers[admin]
var done sync.WaitGroup
errChn := make(chan error, 2)
// Collect volumes
if collectVolumes {
done.Add(1)
go func() {
defer done.Done()
volumes, err := c.service.GetVolumes(provider)
if err != nil {
errChn <- err
}
for tenantId, volumeCount := range volumes {
tenantName := c.allTenants[tenantId]
allVolumes[tenantName] = volumeCount
}
}()
}
// Collect snapshots
if collectSnapshots {
done.Add(1)
go func() {
defer done.Done()
snapshots, err := c.service.GetSnapshots(provider)
if err != nil {
errChn <- err
}
for tenantId, snapshotCount := range snapshots {
tenantName := c.allTenants[tenantId]
allSnapshots[tenantName] = snapshotCount
}
}()
}
done.Wait()
close(errChn)
if e := <-errChn; e != nil {
return nil, e
}
}
// Collect limits per each tenant only if not already collected (plugin lifetime scope)
{
var done sync.WaitGroup
errChn := make(chan error, collectTenants.Size())
for _, tenant := range collectTenants.Elements() {
_, found := c.allLimits[tenant]
if collectLimits && !found {
if err := c.authenticate(metricTypes[0], tenant); err != nil {
return nil, err
}
provider := c.providers[tenant]
done.Add(1)
go func(p *gophercloud.ProviderClient, t string) {
defer done.Done()
limits, err := c.service.GetLimits(p)
if err != nil {
errChn <- err
}
c.allLimits[t] = limits
}(provider, tenant)
}
}
done.Wait()
close(errChn)
if e := <-errChn; e != nil {
return nil, e
}
}
metrics := []plugin.MetricType{}
for _, metricType := range metricTypes {
namespace := metricType.Namespace().Strings()
tenant := namespace[3]
// Construct temporary struct to accommodate all gathered metrics
metricContainer := struct {
S types.Snapshots `json:"snapshots"`
V types.Volumes `json:"volumes"`
L types.Limits `json:"limits"`
}{
allSnapshots[tenant],
allVolumes[tenant],
c.allLimits[tenant],
}
// Extract values by namespace from temporary struct and create metrics
metric := plugin.MetricType{
Timestamp_: time.Now(),
Namespace_: metricType.Namespace(),
Data_: ns.GetValueByNamespace(metricContainer, namespace[4:]),
}
metrics = append(metrics, metric)
}
return metrics, nil
}
