func initiailizeMonitoring() {
defer log.Infoln("Initializing autonomic monitoring")
ch_aut_err := chn.GetAutonomicErrChannel()
enableLogReading = cfg.GetAgentAutonomic().EnableLogReading
mtr.Initialize(srv.List())
// Start log reader if needed
if enableLogReading {
lgr.StartLogReader()
log.WithField("logreader", enableLogReading).Debugln("Log reading is enabled")
}
// Get the list of containers (running or not) to monitor
containers, err := container.Docker().Client.ListContainers(true, false, "")
if err != nil {
log.WithField("err", err).Debugln("Error monitoring containers")
ch_aut_err <- err
}
// Start the monitor for each configured service
for _, c := range containers {
info, _ := container.Docker().Client.InspectContainer(c.Id)
status := getContainerStatus(info)
service, err := srv.GetServiceByImage(c.Image)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"image": c.Image,
}).Warningln("Error monitoring service")
} else {
e := evt.Event{
Service: service.Name,
Image: c.Image,
Instance: c.Id,
Status: status,
}
evt.HandleCreateEvent(e)
evt.HanldeStartEvent(e)
mtr.AddInstance(c.Id)
if _, ok := instBuffer[c.Id]; !ok {
instBuffer[c.Id] = instanceMetricBuffer{
cpuInst: utils.BuildBuffer(c_B_SIZE),
cpuSys: utils.BuildBuffer(c_B_SIZE),
}
}
container.Docker().Client.StartMonitorStats(c.Id, statCallBack, ch_mnt_stats_err)
if status == enum.PENDING && enableLogReading {
startMonitorLog(c.Id)
}
}
}
}
func (p *Stop) Run(config Action) error {
var err error
var toStop string
running := config.Instances.Running
if len(running) < 1 {
log.WithField("err", ErrNoContainerToStop).Errorln("Cannot stop running container. Trying with pending ones...")
pending := config.Instances.Pending
if len(pending) < 1 {
log.WithField("err", ErrNoContainerToStop).Errorln("Cannot stop pending container")
return ErrNoContainerToStop
}
toStop = pending[0]
} else {
toStop = running[0]
}
err = container.Docker().Client.StopContainer(toStop, config.Parameters.StopTimeout)
if err != nil {
log.WithField("err", err).Errorln("Cannot stop container ", toStop)
return err
}
return nil
}
func startMonitorLog(id string) {
var optionsLog = dockerclient.LogOptions{Follow: true, Stdout: true, Stderr: true, Tail: 1}
contLog, err := container.Docker().Client.ContainerLogs(id, &optionsLog)
if err != nil {
log.WithField("error", err).Errorln("Cannot start log monitoring on container ", id)
} else {
lgr.StartCollector(contLog)
}
}
func computeTotalResources() {
info, err := container.Docker().Client.Info()
if err != nil {
log.WithField("err", err).Errorln("Error reading total resources")
return
}
resources.CPU.Total = info.NCPU
resources.Memory.Total = info.MemTotal
for i := 0; i < int(resources.CPU.Total); i++ {
resources.CPU.Cores[i] = true
}
}
func ComputeUsedMemory() (int64, error) {
var memory int64
containers, err := container.Docker().Client.ListContainers(false, false, "")
if err != nil {
return 0, err
}
for _, c := range containers {
if _, err := service.GetServiceByImage(c.Image); err == nil {
cData, err := container.Docker().Client.InspectContainer(c.Id)
if err != nil {
return 0, err
}
memory += cData.Config.Memory
}
}
resources.Memory.Used = memory
return memory, nil
}
func ComputeUsedCpus() (int64, error) {
var cpus int64
containers, err := container.Docker().Client.ListContainers(false, false, "")
if err != nil {
return 0, err
}
for _, c := range containers {
if _, err := service.GetServiceByImage(c.Image); err == nil {
cData, err := container.Docker().Client.InspectContainer(c.Id)
if err != nil {
return 0, err
}
cpuset := strings.Split(cData.HostConfig.CpusetCpus, ",")
cpus += int64(len(cpuset))
}
}
resources.CPU.Used = cpus
return cpus, nil
}
func startMonitoring() {
log.Infoln("Running autonomic monitoring")
ch_aut_err := chn.GetAutonomicErrChannel()
container.Docker().Client.StartMonitorEvents(eventCallback, ch_mnt_events_err)
for {
select {
case err := <-ch_mnt_events_err:
log.WithField("err", err).Fatalln("Error monitoring containers events")
ch_aut_err <- err
case err := <-ch_mnt_stats_err:
log.WithField("err", err).Debugln("Error monitoring containers stats")
ch_aut_err <- err
case <-ch_stop:
return
}
}
}
func (p *Remove) Run(config Action) error {
var err error
stopped := config.Instances.Stopped
if len(stopped) < 1 {
log.WithFields(log.Fields{
"service": config.Service,
"err": errNoContainerToRemove,
}).Errorln("Cannot remove container")
return errNoContainerToRemove
}
ch.SetRemovalNotification(true)
toRemove := stopped[0]
// Assumption: I remove only stopped containers; containers have no volume
err = container.Docker().Client.RemoveContainer(toRemove, false, false)
if err != nil {
log.WithFields(log.Fields{
"service": config.Service,
"instance": toRemove,
"err": err,
}).Errorln("Cannot remove container")
return err
}
waitForRemoval()
log.WithFields(log.Fields{
"service": config.Service,
"instance": toRemove,
}).Debugln("Removed container")
return nil
}
// Events are: attach, commit, copy, create, destroy, die, exec_create, exec_start, export, kill, oom, pause, rename, resize, restart, start, stop, top, unpause, update
func eventCallback(event *dockerclient.Event, ec chan error, args ...interface{}) {
log.Debugln("Received event")
// By now we do not handle events with type != container
if event.Type != "container" {
log.WithField("type", event.Type).Debugln("Received event with type different from 'container'")
return
}
service, err := srv.GetServiceByImage(event.From)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"event": event,
}).Warnln("Cannot handle event")
return
}
e := evt.Event{
Service: service.Name,
Image: event.From,
Instance: event.ID,
Type: event.Type,
}
switch event.Status {
case "create":
log.WithField("image", e.Image).Debugln("Received create signal")
evt.HandleCreateEvent(e)
container.Docker().Client.StartMonitorStats(e.Instance, statCallBack, ch_mnt_stats_err)
case "start":
log.WithField("image", e.Image).Debugln("Received start signal")
if _, ok := instBuffer[e.Instance]; !ok {
instBuffer[e.Instance] = instanceMetricBuffer{
cpuInst: utils.BuildBuffer(c_B_SIZE),
cpuSys: utils.BuildBuffer(c_B_SIZE),
}
}
e.Status = enum.PENDING
evt.HanldeStartEvent(e)
mtr.AddInstance(e.Instance)
if enableLogReading {
startMonitorLog(event.ID)
}
case "stop":
log.WithField("image", e.Image).Debugln("Received stop signal")
case "kill":
log.WithField("image", e.Image).Debugln("Received kill signal")
case "die":
log.WithField("image", e.Image).Debugln("Received die signal")
delete(instBuffer, e.Instance)
mtr.RemoveInstance(e.Instance)
evt.HandleStopEvent(e)
case "destroy":
log.WithField("id", e.Instance).Debugln("Received destroy signal")
evt.HandleRemoveEvent(e)
default:
log.WithFields(log.Fields{
"err": "event not handled",
"event": event.Status,
"image": event.From,
}).Debugln("Received unknown signal")
}
log.Debugln("Event handled")
}
func SetServiceInstanceResources(name string, id string) {
var err error
log.Debugln("Setting new instance resources")
// This is needed otherwise dockerclient does not
// return the correct container information
time.Sleep(100 * time.Millisecond)
info, err := container.Docker().Client.InspectContainer(id)
if err != nil {
log.WithFields(log.Fields{
"id": id,
"err": err,
}).Errorln("Error setting instance resources")
}
cpusetcpus := info.HostConfig.CpusetCpus
portBindings, err := container.GetPortBindings(id)
if err != nil {
log.WithFields(log.Fields{
"service": name,
"id": id,
"err": err,
}).Errorln("Error getting instance port binding")
}
log.WithFields(log.Fields{
"service": name,
"cpusetcpus": cpusetcpus,
"portbindings": portBindings,
}).Debugln("New instance respources")
err = CheckAndSetSpecificCores(cpusetcpus, id)
if err != nil {
log.WithFields(log.Fields{
"service": name,
"id": id,
"cpus": cpusetcpus,
"err": err,
}).Errorln("Error assigning CPU resources to new instance")
}
err = AssignSpecifiPortsToService(name, id, portBindings)
if err != nil {
log.WithFields(log.Fields{
"service": name,
"id": id,
"bindings": portBindings,
"err": err,
}).Errorln("Error assigning port resources to new instance")
}
guestPort := service.GetDiscoveryPort(name)
if hostPorts, ok := portBindings[guestPort]; ok {
if len(hostPorts) > 0 {
service.SaveInstanceAddress(id, hostPorts[0])
log.WithFields(log.Fields{
"service": name,
"instance": id,
"guest": guestPort,
"host": hostPorts[0],
}).Debugln("Saved instance address")
} else {
log.WithFields(log.Fields{
"service": name,
"instance": id,
"guest": guestPort,
}).Debugln("Cannot register instance address: host ports < 0")
}
} else {
log.WithFields(log.Fields{
"service": name,
"instance": id,
"guest": guestPort,
}).Debugln("Cannot register instance address: no bindings")
}
}