// Get information about the cgroup subsystems.
func GetCgroupSubsystems() (CgroupSubsystems, error) {
// Get all cgroup mounts.
allCgroups, err := cgroups.GetCgroupMounts()
if err != nil {
return CgroupSubsystems{}, err
}
if len(allCgroups) == 0 {
return CgroupSubsystems{}, fmt.Errorf("failed to find cgroup mounts")
}
// Trim the mounts to only the subsystems we care about.
supportedCgroups := make([]cgroups.Mount, 0, len(allCgroups))
mountPoints := make(map[string]string, len(allCgroups))
for _, mount := range allCgroups {
for _, subsystem := range mount.Subsystems {
if _, ok := supportedSubsystems[subsystem]; ok {
supportedCgroups = append(supportedCgroups, mount)
mountPoints[subsystem] = mount.Mountpoint
}
}
}
return CgroupSubsystems{
Mounts: supportedCgroups,
MountPoints: mountPoints,
}, nil
}
func Register(machineInfoFactory info.MachineInfoFactory) error {
// Get all cgroup mounts.
allCgroups, err := cgroups.GetCgroupMounts()
if err != nil {
return err
}
if len(allCgroups) == 0 {
return fmt.Errorf("failed to find cgroup mounts for the raw factory")
}
// Trim the mounts to only the subsystems we care about.
supportedCgroups := make([]cgroups.Mount, 0, len(allCgroups))
mountPoints := make(map[string]string, len(allCgroups))
for _, mount := range allCgroups {
for _, subsystem := range mount.Subsystems {
if _, ok := supportedSubsystems[subsystem]; ok {
supportedCgroups = append(supportedCgroups, mount)
mountPoints[subsystem] = mount.Mountpoint
}
}
}
if len(supportedCgroups) == 0 {
return fmt.Errorf("failed to find supported cgroup mounts for the raw factory")
}
glog.Infof("Registering Raw factory")
factory := &rawFactory{
machineInfoFactory: machineInfoFactory,
cgroupSubsystems: &cgroupSubsystems{
mounts: supportedCgroups,
mountPoints: mountPoints,
},
}
container.RegisterContainerHandlerFactory(factory)
return nil
}
func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error {
var (
dest = m.Destination
data = label.FormatMountLabel(m.Data, mountLabel)
)
if !strings.HasPrefix(dest, rootfs) {
dest = filepath.Join(rootfs, dest)
}
switch m.Device {
case "proc", "sysfs":
if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
return err
}
return syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), "")
case "mqueue":
if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
return err
}
if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), ""); err != nil {
return err
}
return label.SetFileLabel(dest, mountLabel)
case "tmpfs":
stat, err := os.Stat(dest)
if err != nil {
if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
return err
}
}
if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), data); err != nil {
return err
}
if stat != nil {
if err = os.Chmod(dest, stat.Mode()); err != nil {
return err
}
}
return nil
case "devpts":
if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
return err
}
return syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), data)
case "bind":
stat, err := os.Stat(m.Source)
if err != nil {
// error out if the source of a bind mount does not exist as we will be
// unable to bind anything to it.
return err
}
// ensure that the destination of the bind mount is resolved of symlinks at mount time because
// any previous mounts can invalidate the next mount's destination.
// this can happen when a user specifies mounts within other mounts to cause breakouts or other
// evil stuff to try to escape the container's rootfs.
if dest, err = symlink.FollowSymlinkInScope(filepath.Join(rootfs, m.Destination), rootfs); err != nil {
return err
}
if err := checkMountDestination(rootfs, dest); err != nil {
return err
}
if err := createIfNotExists(dest, stat.IsDir()); err != nil {
return err
}
if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), data); err != nil {
return err
}
if m.Flags&syscall.MS_RDONLY != 0 {
if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags|syscall.MS_REMOUNT), ""); err != nil {
return err
}
}
if m.Relabel != "" {
if err := label.Relabel(m.Source, mountLabel, m.Relabel); err != nil {
return err
}
}
if m.Flags&syscall.MS_PRIVATE != 0 {
if err := syscall.Mount("", dest, "none", uintptr(syscall.MS_PRIVATE), ""); err != nil {
return err
}
}
case "cgroup":
mounts, err := cgroups.GetCgroupMounts()
if err != nil {
return err
}
var binds []*configs.Mount
for _, mm := range mounts {
dir, err := mm.GetThisCgroupDir()
if err != nil {
return err
}
binds = append(binds, &configs.Mount{
Device: "bind",
Source: filepath.Join(mm.Mountpoint, dir),
Destination: filepath.Join(m.Destination, strings.Join(mm.Subsystems, ",")),
Flags: syscall.MS_BIND | syscall.MS_REC | syscall.MS_RDONLY,
})
}
tmpfs := &configs.Mount{
Device: "tmpfs",
Destination: m.Destination,
Flags: syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV,
}
if err := mountToRootfs(tmpfs, rootfs, mountLabel); err != nil {
return err
}
for _, b := range binds {
if err := mountToRootfs(b, rootfs, mountLabel); err != nil {
return err
}
}
default:
return fmt.Errorf("unknown mount device %q to %q", m.Device, m.Destination)
}
return nil
}
func (e *Exporter) collect(ch chan<- prometheus.Metric) error {
containers, err := e.manager.Containers()
if err != nil {
e.errors.WithLabelValues("list").Inc()
return err
}
mounts, err := cgroups.GetCgroupMounts()
if err != nil {
return err
}
for _, container := range containers {
cgroupPaths := make(map[string]string, len(mounts))
for _, mount := range mounts {
p, err := e.findCgroupPath(mount.Mountpoint, container.ID)
if err != nil {
continue // Ignore
}
for _, subsystem := range mount.Subsystems {
cgroupPaths[subsystem] = p
}
}
manager := &fs.Manager{
Paths: cgroupPaths,
Cgroups: nil,
}
stats, err := manager.GetStats()
if err != nil {
e.errors.WithLabelValues("stats").Inc()
return err
}
name := container.Name
id := container.ID
image := container.Image
//Additional information
cont, err := e.client.InspectContainer(container.ID)
if err != nil {
e.errors.WithLabelValues("dockerclient").Inc()
return err
}
var labelValues = make([]string, len(e.labels))
for index, labelName := range e.labels {
if val, ok := cont.Config.Labels[labelName]; ok {
labelValues[index] = val
} else {
labelValues[index] = ""
}
}
// Last seen
e.lastSeen.WithLabelValues(append([]string{name, id, image}, labelValues...)...).Set(float64(time.Now().Unix()))
// CPU stats
// - Usage
for i, value := range stats.CpuStats.CpuUsage.PercpuUsage {
e.cpuUsageSecondsPerCPU.WithLabelValues(append([]string{name, id, image, fmt.Sprintf("cpu%02d", i)}, labelValues...)...).Set(float64(value) / float64(time.Second))
}
e.cpuUsageSeconds.WithLabelValues(append([]string{name, id, image, "kernel"}, labelValues...)...).Set(float64(stats.CpuStats.CpuUsage.UsageInKernelmode) / float64(time.Second))
e.cpuUsageSeconds.WithLabelValues(append([]string{name, id, image, "user"}, labelValues...)...).Set(float64(stats.CpuStats.CpuUsage.UsageInUsermode) / float64(time.Second))
// - Throttling
e.cpuThrottledPeriods.WithLabelValues(append([]string{name, id, image, "total"}, labelValues...)...).Set(float64(stats.CpuStats.ThrottlingData.Periods))
e.cpuThrottledPeriods.WithLabelValues(append([]string{name, id, image, "throttled"}, labelValues...)...).Set(float64(stats.CpuStats.ThrottlingData.ThrottledPeriods))
e.cpuThrottledTime.WithLabelValues(append([]string{name, id, image}, labelValues...)...).Set(float64(stats.CpuStats.ThrottlingData.ThrottledTime) / float64(time.Second))
// Memory stats
e.memoryUsageBytes.WithLabelValues(append([]string{name, id, image}, labelValues...)...).Set(float64(stats.MemoryStats.Usage.Usage))
e.memoryMaxUsageBytes.WithLabelValues(append([]string{name, id, image}, labelValues...)...).Set(float64(stats.MemoryStats.Usage.MaxUsage))
for t, value := range stats.MemoryStats.Stats {
if isMemoryPagingCounter(t) {
e.memoryPaging.WithLabelValues(append([]string{name, id, image, t}, labelValues...)...).Set(float64(value))
} else {
e.memoryStats.WithLabelValues(append([]string{name, id, image, t}, labelValues...)...).Set(float64(value))
}
}
e.memoryFailures.WithLabelValues(append([]string{name, id, image}, labelValues...)...).Set(float64(stats.MemoryStats.Usage.Failcnt))
// BlkioStats
devMap, err := newDeviceMap(procDiskStats)
if err != nil {
return err
}
for _, stat := range stats.BlkioStats.IoServiceBytesRecursive {
e.blkioIoServiceBytesRecursive.WithLabelValues(append([]string{name, id, image, devMap.name(stat.Major, stat.Minor), stat.Op}, labelValues...)...).Set(float64(stat.Value))
}
for _, stat := range stats.BlkioStats.IoServicedRecursive {
e.blkioIoServicedRecursive.WithLabelValues(append([]string{name, id, image, devMap.name(stat.Major, stat.Minor), stat.Op}, labelValues...)...).Set(float64(stat.Value))
}
for _, stat := range stats.BlkioStats.IoQueuedRecursive {
e.blkioIoQueuedRecursive.WithLabelValues(append([]string{name, id, image, devMap.name(stat.Major, stat.Minor), stat.Op}, labelValues...)...).Set(float64(stat.Value))
}
for _, stat := range stats.BlkioStats.SectorsRecursive {
e.blkioSectorsRecursive.WithLabelValues(append([]string{name, id, image, devMap.name(stat.Major, stat.Minor)}, labelValues...)...).Set(float64(stat.Value))
}
}
return nil
}
func mountToRootfs(m *configs.Mount, rootfs, mountLabel string) error {
var (
dest = m.Destination
data = label.FormatMountLabel(m.Data, mountLabel)
)
if !strings.HasPrefix(dest, rootfs) {
dest = filepath.Join(rootfs, dest)
}
switch m.Device {
case "proc", "sysfs":
if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
return err
}
return syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), "")
case "mqueue":
if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
return err
}
if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), ""); err != nil {
return err
}
return label.SetFileLabel(dest, mountLabel)
case "tmpfs":
stat, err := os.Stat(dest)
if err != nil {
if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
return err
}
}
if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), data); err != nil {
return err
}
if stat != nil {
if err = os.Chmod(dest, stat.Mode()); err != nil {
return err
}
}
return nil
case "devpts":
if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
return err
}
return syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), data)
case "bind":
stat, err := os.Stat(m.Source)
if err != nil {
// error out if the source of a bind mount does not exist as we will be
// unable to bind anything to it.
return err
}
if err := createIfNotExists(dest, stat.IsDir()); err != nil {
return err
}
if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags), data); err != nil {
return err
}
if m.Flags&syscall.MS_RDONLY != 0 {
if err := syscall.Mount(m.Source, dest, m.Device, uintptr(m.Flags|syscall.MS_REMOUNT), ""); err != nil {
return err
}
}
if m.Relabel != "" {
if err := label.Relabel(m.Source, mountLabel, m.Relabel); err != nil {
return err
}
}
if m.Flags&syscall.MS_PRIVATE != 0 {
if err := syscall.Mount("", dest, "none", uintptr(syscall.MS_PRIVATE), ""); err != nil {
return err
}
}
case "cgroup":
mounts, err := cgroups.GetCgroupMounts()
if err != nil {
return err
}
var binds []*configs.Mount
for _, mm := range mounts {
dir, err := mm.GetThisCgroupDir()
if err != nil {
return err
}
binds = append(binds, &configs.Mount{
Device: "bind",
Source: filepath.Join(mm.Mountpoint, dir),
Destination: filepath.Join(m.Destination, strings.Join(mm.Subsystems, ",")),
Flags: syscall.MS_BIND | syscall.MS_REC | syscall.MS_RDONLY,
})
}
tmpfs := &configs.Mount{
Device: "tmpfs",
Destination: m.Destination,
Flags: syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV,
}
if err := mountToRootfs(tmpfs, rootfs, mountLabel); err != nil {
return err
}
for _, b := range binds {
if err := mountToRootfs(b, rootfs, mountLabel); err != nil {
return err
}
}
default:
return fmt.Errorf("unknown mount device %q to %q", m.Device, m.Destination)
}
return nil
}