Beispiel #1
0
func updateMetric(target string, metricType enum.MetricType, metric string, toAdd []float64) {
	defer mutex_instMet.Unlock()

	mutex_instMet.Lock()
	switch metricType {
	case enum.METRIC_T_BASE:
		if toUpdateInstace, ok := instancesMetrics[target]; ok {
			values := toUpdateInstace.BaseMetrics[metric]
			values = append(values, toAdd...)
			toUpdateInstace.BaseMetrics[metric] = values
		} else {
			log.WithFields(log.Fields{
				"target": target,
				"metric": metric,
			}).Errorln("Cannot update instance metric: unknown instance")
		}
	case enum.METRIC_T_USER:
		if toUpdateService, ok := servicesMetrics[target]; ok {
			values := toUpdateService.UserMetrics[metric]
			values = append(values, toAdd...)
			toUpdateService.UserMetrics[metric] = values
		} else {
			log.WithFields(log.Fields{
				"target": target,
				"metric": metric,
			}).Errorln("Cannot update service metric: unknown service")
		}
	}
}
Beispiel #2
0
func AssignSpecifiPortsToService(name string, id string, ports map[string][]string) error {
	defer runtime.Gosched()
	mutex_port.Lock()
	servicePorts := resources.Network.ServicePorts[name]

	// Check if ALL the bindings are possible
	// Otherwise "abort" the operation
	for guest, bindings := range ports {
		status := servicePorts.Status[guest]
		for _, binding := range bindings {
			if contains(binding, status.Occupied) {
				servicePorts.LastAssigned = make(map[string][]string)
				mutex_port.Unlock()
				return errPortAlreadyOccupied
			}
		}
	}

	for guest, bindings := range ports {
		status := servicePorts.Status[guest]
		for _, binding := range bindings {
			if contains(binding, status.Available) {
				status.Available, status.Occupied = moveSpecificItem(binding, status.Available, status.Occupied)
			} else {
				log.WithFields(log.Fields{
					"service": name,
					"guest":   guest,
					"port":    binding,
				}).Warnln("Cannot find port in available ones. Adding it to occupied")
				status.Occupied = append(status.Occupied, binding)
			}
		}

		servicePorts.Status[guest] = status
		servicePorts.LastAssigned = ports
	}

	resources.Network.ServicePorts[name] = servicePorts
	instanceBindings[id] = ports

	mutex_port.Unlock()

	log.WithFields(log.Fields{
		"service":  name,
		"instance": id,
		"ports":    ports,
	}).Debugln("Assigned ports to service instance")
	return nil
}
Beispiel #3
0
func readCommand(r *http.Request) (Command, error) {
	var err error
	var cmd Command

	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		log.WithField("err", err).Errorln("Error reading command body")
		return Command{}, err
	}

	if err = r.Body.Close(); err != nil {
		log.WithField("err", err).Errorln("Error closing command body")
		return Command{}, err
	}

	if err = json.Unmarshal(body, &cmd); err != nil {
		log.WithField("err", err).Errorln("Error unmarshaling command body")
		return Command{}, err
	}

	cmd.Timestamp = time.Now()

	log.WithFields(log.Fields{
		"name":      cmd.Name,
		"target":    cmd.Target,
		"timestamp": cmd.Timestamp,
	}).Debugln("Received command")

	return cmd, nil
}
Beispiel #4
0
func RequestPortsForService(name string) (map[string]string, error) {
	defer runtime.Gosched()
	mutex_port.Lock()
	servicePorts := resources.Network.ServicePorts[name]

	// Check if all the ports can be assigned
	// otherwise "abort" the operation
	for _, host := range servicePorts.Status {
		if len(host.Available) < 1 {
			servicePorts.LastRequested = make(map[string]string)
			resources.Network.ServicePorts[name] = servicePorts
			mutex_port.Unlock()
			return make(map[string]string), ErrNoAvailablePorts
		}
	}

	requestedPorts := make(map[string]string)
	for guest, host := range servicePorts.Status {
		requestedPorts[guest] = host.Available[len(host.Available)-1]
	}
	servicePorts.LastRequested = requestedPorts
	resources.Network.ServicePorts[name] = servicePorts

	mutex_port.Unlock()

	log.WithFields(log.Fields{
		"service":        name,
		"requestedPorts": requestedPorts,
	}).Debugln("Requested ports for service")
	return requestedPorts, nil
}
Beispiel #5
0
func stopInstance(name string, instance string) {
	log.Debugln("Changing instance status to stop...")
	status := srv.GetServiceInstanceStatus(name, instance)
	if status == enum.STOPPED {
		log.Debugln("Instance already stopped")
		return
	}

	err := srv.ChangeServiceInstanceStatus(name, instance, status, enum.STOPPED)
	if err != nil {
		log.WithField("err", err).Errorln("Cannot stop service instance")
		return
	}

	log.Debugln("Updating events...")
	evt_mutex.Lock()
	srvEvents := events.Service[name]
	srvEvents.Stop = append(srvEvents.Stop, instance)
	events.Service[name] = srvEvents
	evt_mutex.Unlock()

	log.Debugln("Unregistering instance...")
	srv.UnregisterServiceInstance(name, instance)

	log.WithFields(log.Fields{
		"service": name,
		"id":      instance,
	}).Infoln("stopped instance")
}
Beispiel #6
0
func computeSysMetrics(instMetrics map[string]data.MetricData) data.MetricData {
	// TODO - improve by adding capacity
	baseMetrics := make(map[string]float64)
	cpuSys := 0.0
	memSys := make([]float64, 0, len(instMetrics))
	for instance, metrics := range instMetrics {
		service, err := srv.GetServiceById(instance)
		if err != nil {
			log.WithFields(log.Fields{
				"instance": instance,
			}).Errorln("Cannot find service by instance")
		} else {
			instCpus := service.Docker.CPUnumber
			instCpuValue := metrics.BaseMetrics[enum.METRIC_CPU_AVG.ToString()] * float64(instCpus)
			// CPU
			cpuSys += instCpuValue

			// MEM
			// TODO
		}
	}

	baseMetrics[enum.METRIC_CPU_AVG.ToString()] = cpuSys / float64(res.GetResources().CPU.Total)
	baseMetrics[enum.METRIC_MEM_AVG.ToString()] = utils.Mean(memSys)
	sysMetrics := data.MetricData{
		BaseMetrics: baseMetrics,
	}

	return sysMetrics

}
Beispiel #7
0
func GetCoresAvailable(number int) (string, bool) {
	defer runtime.Gosched()

	cores_str := make([]string, 0, number)
	mutex_cpu.RLock()
	for i := 0; i < len(resources.CPU.Cores); i++ {
		if resources.CPU.Cores[i] == true {
			cores_str = append(cores_str, strconv.Itoa(i))
		}

		if len(cores_str) >= number {
			break
		}
	}

	if len(cores_str) < number {
		log.Debugln("Error getting available cores: number of free cores < ", number)
		mutex_cpu.RUnlock()
		return "", false
	}

	mutex_cpu.RUnlock()

	cores := strings.Join(cores_str, ",")
	log.WithFields(log.Fields{
		"number": number,
		"cores":  cores,
	}).Debugln("Getting available cores")
	return cores, true
}
Beispiel #8
0
func keepAlive(name string, id string) {
	var err error
	discoveryConf := cfg.GetAgentDiscovery()
	ticker := time.NewTicker(time.Duration(discoveryConf.TTL-1) * time.Second)
	opt := discovery.Options{
		"TTL": time.Duration(discoveryConf.TTL) * time.Second,
	}

	ch_stop := ch.CreateInstanceChannel(id)

	isntanceKey := discoveryConf.AppRoot + "/" + name + "/" + id
	instanceValue := addressMap[id]

	for {
		select {
		case <-ticker.C:
			err = discovery.Set(isntanceKey, instanceValue, opt)
			if err != nil {
				log.WithFields(log.Fields{
					"service":  name,
					"instance": id,
					"address":  instanceValue,
					"err":      err,
				}).Errorln("Error keeping instance alive")
			}
		case <-ch_stop:
			log.Debugln("Stopping keep alive routine")
			return
		}
	}
}
Beispiel #9
0
func updateNodeResources() {
	res.ComputeUsedResources()

	log.WithFields(log.Fields{
		"totalcpu": res.GetResources().CPU.Total,
		"usedcpu":  res.GetResources().CPU.Used,
		"totalmem": res.GetResources().Memory.Total,
		"usedmem":  res.GetResources().Memory.Used,
	}).Debugln("Updated node resources")
}
Beispiel #10
0
func executeActions(target *cfg.Service, actions []enum.Action) {
	var err error
	for _, actionType := range actions {
		config := buildConfig(target, actionType)
		actExecutor := action.Get(actionType)
		err = actExecutor.Run(config)
		if err != nil {
			log.WithFields(log.Fields{
				"err":    err,
				"action": actionType.ToString(),
			}).Errorln("Action not executed")
		}

		log.WithFields(log.Fields{
			"target": target.Name,
			"action": actionType.ToString(),
		}).Infoln("Action executed")
	}
}
Beispiel #11
0
func displayPolicy(chosenPolicy *data.Policy) {
	targets := make([]string, 0, len(chosenPolicy.Targets))
	for _, target := range chosenPolicy.Targets {
		targets = append(targets, target)
	}

	log.WithFields(log.Fields{
		"name":    chosenPolicy.Name,
		"weight":  fmt.Sprintf("%.2f", chosenPolicy.Weight),
		"targets": targets,
	}).Infoln("Policy to actuate")
}
Beispiel #12
0
func initializeStorage() {
	_, err := storage.New(cfg.GetAgentStorage().StorageService)
	if err != nil {
		log.WithFields(log.Fields{
			"status":  "warning",
			"error":   err,
			"default": storage.Name(),
		}).Warnln("Error initializing storage service")
	} else {
		log.WithField(storage.Name(), "ok").Infoln("Storage service initialized")
	}
}
Beispiel #13
0
func initializeMetricSerivice() {
	_, err := metric.New(cfg.GetAgentMetric().MetricService, cfg.GetAgentMetric().Configuration)
	if err != nil {
		log.WithFields(log.Fields{
			"status":  "warning",
			"error":   err,
			"default": metric.Name(),
		}).Warnln("Error initializing metric service")
	} else {
		log.WithField(metric.Name(), "ok").Infoln("Metric service initialized")
	}
}
Beispiel #14
0
func initiailizeMonitoring() {
	defer log.Infoln("Initializing autonomic monitoring")
	ch_aut_err := chn.GetAutonomicErrChannel()
	enableLogReading = cfg.GetAgentAutonomic().EnableLogReading
	mtr.Initialize(srv.List())

	// Start log reader if needed
	if enableLogReading {
		lgr.StartLogReader()
		log.WithField("logreader", enableLogReading).Debugln("Log reading is enabled")
	}

	// Get the list of containers (running or not) to monitor
	containers, err := container.Docker().Client.ListContainers(true, false, "")
	if err != nil {
		log.WithField("err", err).Debugln("Error monitoring containers")
		ch_aut_err <- err
	}

	// Start the monitor for each configured service
	for _, c := range containers {
		info, _ := container.Docker().Client.InspectContainer(c.Id)
		status := getContainerStatus(info)
		service, err := srv.GetServiceByImage(c.Image)
		if err != nil {
			log.WithFields(log.Fields{
				"err":   err,
				"image": c.Image,
			}).Warningln("Error monitoring service")
		} else {
			e := evt.Event{
				Service:  service.Name,
				Image:    c.Image,
				Instance: c.Id,
				Status:   status,
			}

			evt.HandleCreateEvent(e)
			evt.HanldeStartEvent(e)
			mtr.AddInstance(c.Id)
			if _, ok := instBuffer[c.Id]; !ok {
				instBuffer[c.Id] = instanceMetricBuffer{
					cpuInst: utils.BuildBuffer(c_B_SIZE),
					cpuSys:  utils.BuildBuffer(c_B_SIZE),
				}
			}
			container.Docker().Client.StartMonitorStats(c.Id, statCallBack, ch_mnt_stats_err)
			if status == enum.PENDING && enableLogReading {
				startMonitorLog(c.Id)
			}
		}
	}
}
Beispiel #15
0
func FreePortsFromService(name string, id string) {
	defer runtime.Gosched()
	mutex_port.Lock()
	servicePorts := resources.Network.ServicePorts[name]
	if ports, ok := instanceBindings[id]; ok {
		for guest, bindings := range ports {
			status := servicePorts.Status[guest]
			for _, binding := range bindings {
				if contains(binding, status.Occupied) {
					status.Occupied, status.Available = moveSpecificItem(binding, status.Occupied, status.Available)
				} else {
					log.WithFields(log.Fields{
						"service":  name,
						"instance": id,
						"guest":    guest,
						"host":     binding,
					}).Warnln("Cannot find port in occupied list")
				}

			}

			servicePorts.Status[guest] = status
		}

		resources.Network.ServicePorts[name] = servicePorts
		delete(instanceBindings, id)
		log.WithFields(log.Fields{
			"service":  name,
			"instance": id,
			"ports":    ports,
		}).Debugln("Released ports of service instance")
	} else {
		log.WithFields(log.Fields{
			"service":  name,
			"instance": id,
		}).Warnln("Cannot find instance id in port map")
	}

	mutex_port.Unlock()
}
Beispiel #16
0
// /gru/v1/analytics/system
func GetAnalyticsSystem(w http.ResponseWriter, r *http.Request) {
	analytics := analyzer.GetSystemAnalytics()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(analytics); err != nil {
		log.WithFields(log.Fields{
			"status":  "http response",
			"request": "GetAnalyticsSystem",
			"error":   err,
		}).Errorln("API Server")
	}
}
Beispiel #17
0
func displayStatsOfServices(stats data.GruStats) {
	for name, value := range stats.Metrics.Service {
		service, _ := srv.GetServiceByName(name)
		log.WithFields(log.Fields{
			"pending:": len(service.Instances.Pending),
			"running:": len(service.Instances.Running),
			"stopped:": len(service.Instances.Stopped),
			"paused:":  len(service.Instances.Paused),
			"cpu avg":  fmt.Sprintf("%.2f", value.BaseMetrics[enum.METRIC_CPU_AVG.ToString()]),
			"mem avg":  fmt.Sprintf("%.2f", value.BaseMetrics[enum.METRIC_MEM_AVG.ToString()]),
		}).Infoln("Stats computed: ", name)
	}
}
Beispiel #18
0
// /gru/v1/actions
func GetInfoActions(w http.ResponseWriter, r *http.Request) {
	actions := action.List()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(actions); err != nil {
		log.WithFields(log.Fields{
			"status":  "http response",
			"request": "GetInfoPolicies",
			"error":   err,
		}).Errorln("API Server")
	}
}
Beispiel #19
0
// /gru/v1/shared/
func GetSharedData(w http.ResponseWriter, r *http.Request) {
	info, _ := data.GetSharedLocal()

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(info); err != nil {
		log.WithFields(log.Fields{
			"status":  "http response",
			"request": "GetInfoNode",
			"error":   err,
		}).Errorln("API Server")
	}
}
Beispiel #20
0
func GetPortBindings(id string) (map[string][]string, error) {
	info, err := docker.Client.InspectContainer(id)
	if err != nil {
		log.WithFields(log.Fields{
			"id":  id,
			"err": err,
		}).Errorln("Error inspecting instance")
	}

	portBindings := createPortBindings(info.HostConfig.PortBindings)

	return portBindings, err
}
Beispiel #21
0
func ComputeMetricAnalytics(service string, metrics map[string]float64) map[string]float64 {
	expressions := cfg.GetAnalyticExpr()
	srvExprList := srv.GetServiceAnalyticsExprList(service)
	srvConstraints := srv.GetServiceConstraints(service)
	metricAnalytics := make(map[string]float64, len(expressions))

	for _, expr := range srvExprList {
		if curExpr, ok := expressions[expr]; ok {
			log.WithField("expr", expr).Debugln("Evaluating expression")
			toEval := buildExpression(curExpr, metrics, srvConstraints)
			result, err := evaler.Eval(toEval)
			if err != nil {
				log.WithFields(log.Fields{
					"err":  err,
					"expr": toEval,
				}).Warnln("Error evaluating expression")

				metricAnalytics[expr] = 0.0
			} else {
				value := evaler.BigratToFloat(result)
				value = math.Min(value, 1.0)
				value = math.Max(value, 0.0)
				metricAnalytics[expr] = value
			}

			log.WithFields(log.Fields{
				"service": service,
				"expr":    expr,
				"value":   metricAnalytics[expr],
			}).Debugln("Expression evaluated")

		} else {
			log.WithField("expr", expr).Errorln("Cannot compute expression: expression unknown")
		}
	}

	return metricAnalytics
}
Beispiel #22
0
func GetDiscoveryPort(name string) string {
	srv, err := GetServiceByName(name)
	if err != nil {
		log.WithFields(log.Fields{
			"err":     err,
			"service": name,
		}).Debugln("Error getting discovery port")

		return ""
	}

	return srv.DiscoveryPort

}
Beispiel #23
0
func (p *Remove) Run(config Action) error {
	var err error
	stopped := config.Instances.Stopped
	if len(stopped) < 1 {
		log.WithFields(log.Fields{
			"service": config.Service,
			"err":     errNoContainerToRemove,
		}).Errorln("Cannot remove container")

		return errNoContainerToRemove
	}

	ch.SetRemovalNotification(true)

	toRemove := stopped[0]
	// Assumption: I remove only stopped containers; containers have no volume
	err = container.Docker().Client.RemoveContainer(toRemove, false, false)
	if err != nil {
		log.WithFields(log.Fields{
			"service":  config.Service,
			"instance": toRemove,
			"err":      err,
		}).Errorln("Cannot remove container")

		return err
	}

	waitForRemoval()

	log.WithFields(log.Fields{
		"service":  config.Service,
		"instance": toRemove,
	}).Debugln("Removed container")

	return nil
}
Beispiel #24
0
func addValue(entry logEntry) {
	if entry.value < 0.0 {
		log.WithFields(log.Fields{
			"service": entry.service,
			"metric":  entry.metric,
			"value":   entry.value,
		}).Warnln("Metric value < 0")
		return
	}

	values := make([]float64, 0, 1)
	values = append(values, entry.value)
	mtr.UpdateUserMetric(entry.service, entry.metric, values)

}
Beispiel #25
0
func InitializeServiceAvailablePorts(name string, ports map[string]string) {
	defer runtime.Gosched()
	mutex_port.Lock()

	servicePorts := resources.Network.ServicePorts[name]

	for guest, host := range ports {
		servicePorts.LastAssigned = make(map[string][]string)
		servicePorts.Status = make(map[string]PortStatus)

		hostRange, err := utils.GetCompleteRange(host)
		if err != nil {
			log.WithFields(log.Fields{
				"err":     err,
				"service": name,
				"guest":   guest,
				"host":    host,
			}).Warnln("Cannot compute host port range for guest port")
		}

		status := PortStatus{
			Available: hostRange,
			Occupied:  []string{},
		}

		servicePorts.Status[guest] = status
	}
	resources.Network.ServicePorts[name] = servicePorts

	mutex_port.Unlock()

	log.WithFields(log.Fields{
		"service": name,
		"ports":   ports,
	}).Debugln("Initialed service ports")
}
Beispiel #26
0
func UnregisterServiceInstance(name string, id string) {
	discoveryConf := cfg.GetAgentDiscovery()
	isntanceKey := discoveryConf.AppRoot + "/" + name + "/" + id
	discovery.Delete(isntanceKey)
	ch_stop, err := ch.GetInstanceChannel(id)
	if err != nil {
		log.WithFields(log.Fields{
			"service":  name,
			"instance": id,
			"err":      err,
		}).Errorln("Error getting instance stop channel")
		return
	}
	ch_stop <- struct{}{}
	ch.RemoveInstanceChannel(id)
}
Beispiel #27
0
func executeCommand(cmd Command) {
	switch cmd.Name {
	case "start":
		startCommand(cmd)
	case "stop":
		stopCommand(cmd)
	case "update":
		updateCommand(cmd)
	default:
		log.Errorln("Unrecognized command name: ", cmd.Name)
	}

	log.WithFields(log.Fields{
		"cmd":    cmd.Name,
		"target": cmd.Target,
	}).Debugln("Executed command")
}
Beispiel #28
0
func TestLogstashFormatter(t *testing.T) {
	assert := assert.New(t)

	lf := LogstashFormatter{Type: "abc"}

	fields := logrus.Fields{
		"message": "def",
		"level":   "ijk",
		"type":    "lmn",
		"one":     1,
		"pi":      3.14,
		"bool":    true,
	}

	entry := logrus.WithFields(fields)
	entry.Message = "msg"
	entry.Level = logrus.InfoLevel

	b, _ := lf.Format(entry)

	var data map[string]interface{}
	dec := json.NewDecoder(bytes.NewReader(b))
	dec.UseNumber()
	dec.Decode(&data)

	// base fields
	assert.Equal(json.Number("1"), data["@version"])
	assert.NotEmpty(data["@timestamp"])
	assert.Equal("abc", data["type"])
	assert.Equal("msg", data["message"])
	assert.Equal("info", data["level"])

	// substituted fields
	assert.Equal("def", data["fields.message"])
	assert.Equal("ijk", data["fields.level"])
	assert.Equal("lmn", data["fields.type"])

	// formats
	assert.Equal(json.Number("1"), data["one"])
	assert.Equal(json.Number("3.14"), data["pi"])
	assert.Equal(true, data["bool"])
}
Beispiel #29
0
func New(name string, conf map[string]interface{}) (metricService, error) {
	metricServ = 0
	for index, mtrc := range metricServices {
		if mtrc.Name() == name {
			err := mtrc.Initialize(conf)
			if err != nil {
				log.WithFields(log.Fields{
					"err":     err,
					"service": mtrc.Name(),
				}).Errorln("Error initializing metric service")
				return metricServices[metricServ], err
			}
			metricServ = index
			log.WithField("service", name).Debugln("Initialized metric service")
			return metricServices[metricServ], nil
		}
	}

	return metricServices[metricServ], ErrNotSupported
}
Beispiel #30
0
func RegisterServiceInstanceId(name string, id string) {
	var err error
	discoveryConf := cfg.GetAgentDiscovery()
	opt := discovery.Options{
		"TTL": time.Duration(discoveryConf.TTL) * time.Second,
	}

	isntanceKey := discoveryConf.AppRoot + "/" + name + "/" + id
	instanceValue := addressMap[id]

	err = discovery.Set(isntanceKey, instanceValue, opt)
	if err != nil {
		log.WithFields(log.Fields{
			"service":  name,
			"instance": id,
			"address":  instanceValue,
			"err":      err,
		}).Errorln("Error registering service instance to discovery service")
	}
}