Exemplo n.º 1
0
func initiailizeMonitoring() {
	defer log.Infoln("Initializing autonomic monitoring")
	ch_aut_err := chn.GetAutonomicErrChannel()
	enableLogReading = cfg.GetAgentAutonomic().EnableLogReading
	mtr.Initialize(srv.List())

	// Start log reader if needed
	if enableLogReading {
		lgr.StartLogReader()
		log.WithField("logreader", enableLogReading).Debugln("Log reading is enabled")
	}

	// Get the list of containers (running or not) to monitor
	containers, err := container.Docker().Client.ListContainers(true, false, "")
	if err != nil {
		log.WithField("err", err).Debugln("Error monitoring containers")
		ch_aut_err <- err
	}

	// Start the monitor for each configured service
	for _, c := range containers {
		info, _ := container.Docker().Client.InspectContainer(c.Id)
		status := getContainerStatus(info)
		service, err := srv.GetServiceByImage(c.Image)
		if err != nil {
			log.WithFields(log.Fields{
				"err":   err,
				"image": c.Image,
			}).Warningln("Error monitoring service")
		} else {
			e := evt.Event{
				Service:  service.Name,
				Image:    c.Image,
				Instance: c.Id,
				Status:   status,
			}

			evt.HandleCreateEvent(e)
			evt.HanldeStartEvent(e)
			mtr.AddInstance(c.Id)
			if _, ok := instBuffer[c.Id]; !ok {
				instBuffer[c.Id] = instanceMetricBuffer{
					cpuInst: utils.BuildBuffer(c_B_SIZE),
					cpuSys:  utils.BuildBuffer(c_B_SIZE),
				}
			}
			container.Docker().Client.StartMonitorStats(c.Id, statCallBack, ch_mnt_stats_err)
			if status == enum.PENDING && enableLogReading {
				startMonitorLog(c.Id)
			}
		}
	}
}
Exemplo n.º 2
0
func (p *Stop) Run(config Action) error {
	var err error
	var toStop string
	running := config.Instances.Running

	if len(running) < 1 {
		log.WithField("err", ErrNoContainerToStop).Errorln("Cannot stop running container. Trying with pending ones...")

		pending := config.Instances.Pending
		if len(pending) < 1 {
			log.WithField("err", ErrNoContainerToStop).Errorln("Cannot stop pending container")
			return ErrNoContainerToStop
		}

		toStop = pending[0]

	} else {

		toStop = running[0]
	}

	err = container.Docker().Client.StopContainer(toStop, config.Parameters.StopTimeout)
	if err != nil {
		log.WithField("err", err).Errorln("Cannot stop container ", toStop)
		return err
	}

	return nil
}
Exemplo n.º 3
0
func startMonitorLog(id string) {
	var optionsLog = dockerclient.LogOptions{Follow: true, Stdout: true, Stderr: true, Tail: 1}
	contLog, err := container.Docker().Client.ContainerLogs(id, &optionsLog)
	if err != nil {
		log.WithField("error", err).Errorln("Cannot start log monitoring on container ", id)
	} else {
		lgr.StartCollector(contLog)
	}
}
Exemplo n.º 4
0
func computeTotalResources() {
	info, err := container.Docker().Client.Info()
	if err != nil {
		log.WithField("err", err).Errorln("Error reading total resources")
		return
	}

	resources.CPU.Total = info.NCPU
	resources.Memory.Total = info.MemTotal

	for i := 0; i < int(resources.CPU.Total); i++ {
		resources.CPU.Cores[i] = true
	}
}
Exemplo n.º 5
0
func ComputeUsedMemory() (int64, error) {
	var memory int64

	containers, err := container.Docker().Client.ListContainers(false, false, "")
	if err != nil {
		return 0, err
	}

	for _, c := range containers {
		if _, err := service.GetServiceByImage(c.Image); err == nil {
			cData, err := container.Docker().Client.InspectContainer(c.Id)
			if err != nil {
				return 0, err
			}

			memory += cData.Config.Memory
		}
	}

	resources.Memory.Used = memory

	return memory, nil
}
Exemplo n.º 6
0
func ComputeUsedCpus() (int64, error) {
	var cpus int64

	containers, err := container.Docker().Client.ListContainers(false, false, "")
	if err != nil {
		return 0, err
	}

	for _, c := range containers {
		if _, err := service.GetServiceByImage(c.Image); err == nil {
			cData, err := container.Docker().Client.InspectContainer(c.Id)
			if err != nil {
				return 0, err
			}
			cpuset := strings.Split(cData.HostConfig.CpusetCpus, ",")
			cpus += int64(len(cpuset))
		}
	}

	resources.CPU.Used = cpus

	return cpus, nil
}
Exemplo n.º 7
0
func startMonitoring() {
	log.Infoln("Running autonomic monitoring")
	ch_aut_err := chn.GetAutonomicErrChannel()

	container.Docker().Client.StartMonitorEvents(eventCallback, ch_mnt_events_err)
	for {
		select {
		case err := <-ch_mnt_events_err:
			log.WithField("err", err).Fatalln("Error monitoring containers events")
			ch_aut_err <- err
		case err := <-ch_mnt_stats_err:
			log.WithField("err", err).Debugln("Error monitoring containers stats")
			ch_aut_err <- err
		case <-ch_stop:
			return
		}
	}
}
Exemplo n.º 8
0
func (p *Remove) Run(config Action) error {
	var err error
	stopped := config.Instances.Stopped
	if len(stopped) < 1 {
		log.WithFields(log.Fields{
			"service": config.Service,
			"err":     errNoContainerToRemove,
		}).Errorln("Cannot remove container")

		return errNoContainerToRemove
	}

	ch.SetRemovalNotification(true)

	toRemove := stopped[0]
	// Assumption: I remove only stopped containers; containers have no volume
	err = container.Docker().Client.RemoveContainer(toRemove, false, false)
	if err != nil {
		log.WithFields(log.Fields{
			"service":  config.Service,
			"instance": toRemove,
			"err":      err,
		}).Errorln("Cannot remove container")

		return err
	}

	waitForRemoval()

	log.WithFields(log.Fields{
		"service":  config.Service,
		"instance": toRemove,
	}).Debugln("Removed container")

	return nil
}
Exemplo n.º 9
0
// Events are: attach, commit, copy, create, destroy, die, exec_create, exec_start, export, kill, oom, pause, rename, resize, restart, start, stop, top, unpause, update
func eventCallback(event *dockerclient.Event, ec chan error, args ...interface{}) {
	log.Debugln("Received event")
	// By now we do not handle events with type != container
	if event.Type != "container" {
		log.WithField("type", event.Type).Debugln("Received event with type different from 'container'")
		return
	}

	service, err := srv.GetServiceByImage(event.From)
	if err != nil {
		log.WithFields(log.Fields{
			"err":   err,
			"event": event,
		}).Warnln("Cannot handle event")
		return
	}

	e := evt.Event{
		Service:  service.Name,
		Image:    event.From,
		Instance: event.ID,
		Type:     event.Type,
	}

	switch event.Status {
	case "create":
		log.WithField("image", e.Image).Debugln("Received create signal")
		evt.HandleCreateEvent(e)
		container.Docker().Client.StartMonitorStats(e.Instance, statCallBack, ch_mnt_stats_err)
	case "start":
		log.WithField("image", e.Image).Debugln("Received start signal")
		if _, ok := instBuffer[e.Instance]; !ok {
			instBuffer[e.Instance] = instanceMetricBuffer{
				cpuInst: utils.BuildBuffer(c_B_SIZE),
				cpuSys:  utils.BuildBuffer(c_B_SIZE),
			}
		}
		e.Status = enum.PENDING
		evt.HanldeStartEvent(e)
		mtr.AddInstance(e.Instance)
		if enableLogReading {
			startMonitorLog(event.ID)
		}
	case "stop":
		log.WithField("image", e.Image).Debugln("Received stop signal")
	case "kill":
		log.WithField("image", e.Image).Debugln("Received kill signal")
	case "die":
		log.WithField("image", e.Image).Debugln("Received die signal")
		delete(instBuffer, e.Instance)
		mtr.RemoveInstance(e.Instance)
		evt.HandleStopEvent(e)
	case "destroy":
		log.WithField("id", e.Instance).Debugln("Received destroy signal")
		evt.HandleRemoveEvent(e)
	default:
		log.WithFields(log.Fields{
			"err":   "event not handled",
			"event": event.Status,
			"image": event.From,
		}).Debugln("Received unknown signal")
	}

	log.Debugln("Event handled")

}
Exemplo n.º 10
0
func SetServiceInstanceResources(name string, id string) {
	var err error

	log.Debugln("Setting new instance resources")
	// This is needed otherwise dockerclient does not
	// return the correct container information
	time.Sleep(100 * time.Millisecond)

	info, err := container.Docker().Client.InspectContainer(id)
	if err != nil {
		log.WithFields(log.Fields{
			"id":  id,
			"err": err,
		}).Errorln("Error setting instance resources")
	}

	cpusetcpus := info.HostConfig.CpusetCpus
	portBindings, err := container.GetPortBindings(id)
	if err != nil {
		log.WithFields(log.Fields{
			"service": name,
			"id":      id,
			"err":     err,
		}).Errorln("Error getting instance port binding")
	}

	log.WithFields(log.Fields{
		"service":      name,
		"cpusetcpus":   cpusetcpus,
		"portbindings": portBindings,
	}).Debugln("New instance respources")

	err = CheckAndSetSpecificCores(cpusetcpus, id)
	if err != nil {
		log.WithFields(log.Fields{
			"service": name,
			"id":      id,
			"cpus":    cpusetcpus,
			"err":     err,
		}).Errorln("Error assigning CPU resources to new instance")
	}

	err = AssignSpecifiPortsToService(name, id, portBindings)
	if err != nil {
		log.WithFields(log.Fields{
			"service":  name,
			"id":       id,
			"bindings": portBindings,
			"err":      err,
		}).Errorln("Error assigning port resources to new instance")
	}

	guestPort := service.GetDiscoveryPort(name)
	if hostPorts, ok := portBindings[guestPort]; ok {
		if len(hostPorts) > 0 {
			service.SaveInstanceAddress(id, hostPorts[0])

			log.WithFields(log.Fields{
				"service":  name,
				"instance": id,
				"guest":    guestPort,
				"host":     hostPorts[0],
			}).Debugln("Saved instance address")

		} else {
			log.WithFields(log.Fields{
				"service":  name,
				"instance": id,
				"guest":    guestPort,
			}).Debugln("Cannot register instance address: host ports < 0")
		}

	} else {
		log.WithFields(log.Fields{
			"service":  name,
			"instance": id,
			"guest":    guestPort,
		}).Debugln("Cannot register instance address: no bindings")
	}

}