func updateMetric(target string, metricType enum.MetricType, metric string, toAdd []float64) {
defer mutex_instMet.Unlock()
mutex_instMet.Lock()
switch metricType {
case enum.METRIC_T_BASE:
if toUpdateInstace, ok := instancesMetrics[target]; ok {
values := toUpdateInstace.BaseMetrics[metric]
values = append(values, toAdd...)
toUpdateInstace.BaseMetrics[metric] = values
} else {
log.WithFields(log.Fields{
"target": target,
"metric": metric,
}).Errorln("Cannot update instance metric: unknown instance")
}
case enum.METRIC_T_USER:
if toUpdateService, ok := servicesMetrics[target]; ok {
values := toUpdateService.UserMetrics[metric]
values = append(values, toAdd...)
toUpdateService.UserMetrics[metric] = values
} else {
log.WithFields(log.Fields{
"target": target,
"metric": metric,
}).Errorln("Cannot update service metric: unknown service")
}
}
}
func AssignSpecifiPortsToService(name string, id string, ports map[string][]string) error {
defer runtime.Gosched()
mutex_port.Lock()
servicePorts := resources.Network.ServicePorts[name]
// Check if ALL the bindings are possible
// Otherwise "abort" the operation
for guest, bindings := range ports {
status := servicePorts.Status[guest]
for _, binding := range bindings {
if contains(binding, status.Occupied) {
servicePorts.LastAssigned = make(map[string][]string)
mutex_port.Unlock()
return errPortAlreadyOccupied
}
}
}
for guest, bindings := range ports {
status := servicePorts.Status[guest]
for _, binding := range bindings {
if contains(binding, status.Available) {
status.Available, status.Occupied = moveSpecificItem(binding, status.Available, status.Occupied)
} else {
log.WithFields(log.Fields{
"service": name,
"guest": guest,
"port": binding,
}).Warnln("Cannot find port in available ones. Adding it to occupied")
status.Occupied = append(status.Occupied, binding)
}
}
servicePorts.Status[guest] = status
servicePorts.LastAssigned = ports
}
resources.Network.ServicePorts[name] = servicePorts
instanceBindings[id] = ports
mutex_port.Unlock()
log.WithFields(log.Fields{
"service": name,
"instance": id,
"ports": ports,
}).Debugln("Assigned ports to service instance")
return nil
}
func readCommand(r *http.Request) (Command, error) {
var err error
var cmd Command
body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
if err != nil {
log.WithField("err", err).Errorln("Error reading command body")
return Command{}, err
}
if err = r.Body.Close(); err != nil {
log.WithField("err", err).Errorln("Error closing command body")
return Command{}, err
}
if err = json.Unmarshal(body, &cmd); err != nil {
log.WithField("err", err).Errorln("Error unmarshaling command body")
return Command{}, err
}
cmd.Timestamp = time.Now()
log.WithFields(log.Fields{
"name": cmd.Name,
"target": cmd.Target,
"timestamp": cmd.Timestamp,
}).Debugln("Received command")
return cmd, nil
}
func RequestPortsForService(name string) (map[string]string, error) {
defer runtime.Gosched()
mutex_port.Lock()
servicePorts := resources.Network.ServicePorts[name]
// Check if all the ports can be assigned
// otherwise "abort" the operation
for _, host := range servicePorts.Status {
if len(host.Available) < 1 {
servicePorts.LastRequested = make(map[string]string)
resources.Network.ServicePorts[name] = servicePorts
mutex_port.Unlock()
return make(map[string]string), ErrNoAvailablePorts
}
}
requestedPorts := make(map[string]string)
for guest, host := range servicePorts.Status {
requestedPorts[guest] = host.Available[len(host.Available)-1]
}
servicePorts.LastRequested = requestedPorts
resources.Network.ServicePorts[name] = servicePorts
mutex_port.Unlock()
log.WithFields(log.Fields{
"service": name,
"requestedPorts": requestedPorts,
}).Debugln("Requested ports for service")
return requestedPorts, nil
}
func stopInstance(name string, instance string) {
log.Debugln("Changing instance status to stop...")
status := srv.GetServiceInstanceStatus(name, instance)
if status == enum.STOPPED {
log.Debugln("Instance already stopped")
return
}
err := srv.ChangeServiceInstanceStatus(name, instance, status, enum.STOPPED)
if err != nil {
log.WithField("err", err).Errorln("Cannot stop service instance")
return
}
log.Debugln("Updating events...")
evt_mutex.Lock()
srvEvents := events.Service[name]
srvEvents.Stop = append(srvEvents.Stop, instance)
events.Service[name] = srvEvents
evt_mutex.Unlock()
log.Debugln("Unregistering instance...")
srv.UnregisterServiceInstance(name, instance)
log.WithFields(log.Fields{
"service": name,
"id": instance,
}).Infoln("stopped instance")
}
func computeSysMetrics(instMetrics map[string]data.MetricData) data.MetricData {
// TODO - improve by adding capacity
baseMetrics := make(map[string]float64)
cpuSys := 0.0
memSys := make([]float64, 0, len(instMetrics))
for instance, metrics := range instMetrics {
service, err := srv.GetServiceById(instance)
if err != nil {
log.WithFields(log.Fields{
"instance": instance,
}).Errorln("Cannot find service by instance")
} else {
instCpus := service.Docker.CPUnumber
instCpuValue := metrics.BaseMetrics[enum.METRIC_CPU_AVG.ToString()] * float64(instCpus)
// CPU
cpuSys += instCpuValue
// MEM
// TODO
}
}
baseMetrics[enum.METRIC_CPU_AVG.ToString()] = cpuSys / float64(res.GetResources().CPU.Total)
baseMetrics[enum.METRIC_MEM_AVG.ToString()] = utils.Mean(memSys)
sysMetrics := data.MetricData{
BaseMetrics: baseMetrics,
}
return sysMetrics
}
func GetCoresAvailable(number int) (string, bool) {
defer runtime.Gosched()
cores_str := make([]string, 0, number)
mutex_cpu.RLock()
for i := 0; i < len(resources.CPU.Cores); i++ {
if resources.CPU.Cores[i] == true {
cores_str = append(cores_str, strconv.Itoa(i))
}
if len(cores_str) >= number {
break
}
}
if len(cores_str) < number {
log.Debugln("Error getting available cores: number of free cores < ", number)
mutex_cpu.RUnlock()
return "", false
}
mutex_cpu.RUnlock()
cores := strings.Join(cores_str, ",")
log.WithFields(log.Fields{
"number": number,
"cores": cores,
}).Debugln("Getting available cores")
return cores, true
}
func keepAlive(name string, id string) {
var err error
discoveryConf := cfg.GetAgentDiscovery()
ticker := time.NewTicker(time.Duration(discoveryConf.TTL-1) * time.Second)
opt := discovery.Options{
"TTL": time.Duration(discoveryConf.TTL) * time.Second,
}
ch_stop := ch.CreateInstanceChannel(id)
isntanceKey := discoveryConf.AppRoot + "/" + name + "/" + id
instanceValue := addressMap[id]
for {
select {
case <-ticker.C:
err = discovery.Set(isntanceKey, instanceValue, opt)
if err != nil {
log.WithFields(log.Fields{
"service": name,
"instance": id,
"address": instanceValue,
"err": err,
}).Errorln("Error keeping instance alive")
}
case <-ch_stop:
log.Debugln("Stopping keep alive routine")
return
}
}
}
func updateNodeResources() {
res.ComputeUsedResources()
log.WithFields(log.Fields{
"totalcpu": res.GetResources().CPU.Total,
"usedcpu": res.GetResources().CPU.Used,
"totalmem": res.GetResources().Memory.Total,
"usedmem": res.GetResources().Memory.Used,
}).Debugln("Updated node resources")
}
func executeActions(target *cfg.Service, actions []enum.Action) {
var err error
for _, actionType := range actions {
config := buildConfig(target, actionType)
actExecutor := action.Get(actionType)
err = actExecutor.Run(config)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"action": actionType.ToString(),
}).Errorln("Action not executed")
}
log.WithFields(log.Fields{
"target": target.Name,
"action": actionType.ToString(),
}).Infoln("Action executed")
}
}
func displayPolicy(chosenPolicy *data.Policy) {
targets := make([]string, 0, len(chosenPolicy.Targets))
for _, target := range chosenPolicy.Targets {
targets = append(targets, target)
}
log.WithFields(log.Fields{
"name": chosenPolicy.Name,
"weight": fmt.Sprintf("%.2f", chosenPolicy.Weight),
"targets": targets,
}).Infoln("Policy to actuate")
}
func initializeStorage() {
_, err := storage.New(cfg.GetAgentStorage().StorageService)
if err != nil {
log.WithFields(log.Fields{
"status": "warning",
"error": err,
"default": storage.Name(),
}).Warnln("Error initializing storage service")
} else {
log.WithField(storage.Name(), "ok").Infoln("Storage service initialized")
}
}
func initializeMetricSerivice() {
_, err := metric.New(cfg.GetAgentMetric().MetricService, cfg.GetAgentMetric().Configuration)
if err != nil {
log.WithFields(log.Fields{
"status": "warning",
"error": err,
"default": metric.Name(),
}).Warnln("Error initializing metric service")
} else {
log.WithField(metric.Name(), "ok").Infoln("Metric service initialized")
}
}
func initiailizeMonitoring() {
defer log.Infoln("Initializing autonomic monitoring")
ch_aut_err := chn.GetAutonomicErrChannel()
enableLogReading = cfg.GetAgentAutonomic().EnableLogReading
mtr.Initialize(srv.List())
// Start log reader if needed
if enableLogReading {
lgr.StartLogReader()
log.WithField("logreader", enableLogReading).Debugln("Log reading is enabled")
}
// Get the list of containers (running or not) to monitor
containers, err := container.Docker().Client.ListContainers(true, false, "")
if err != nil {
log.WithField("err", err).Debugln("Error monitoring containers")
ch_aut_err <- err
}
// Start the monitor for each configured service
for _, c := range containers {
info, _ := container.Docker().Client.InspectContainer(c.Id)
status := getContainerStatus(info)
service, err := srv.GetServiceByImage(c.Image)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"image": c.Image,
}).Warningln("Error monitoring service")
} else {
e := evt.Event{
Service: service.Name,
Image: c.Image,
Instance: c.Id,
Status: status,
}
evt.HandleCreateEvent(e)
evt.HanldeStartEvent(e)
mtr.AddInstance(c.Id)
if _, ok := instBuffer[c.Id]; !ok {
instBuffer[c.Id] = instanceMetricBuffer{
cpuInst: utils.BuildBuffer(c_B_SIZE),
cpuSys: utils.BuildBuffer(c_B_SIZE),
}
}
container.Docker().Client.StartMonitorStats(c.Id, statCallBack, ch_mnt_stats_err)
if status == enum.PENDING && enableLogReading {
startMonitorLog(c.Id)
}
}
}
}
func FreePortsFromService(name string, id string) {
defer runtime.Gosched()
mutex_port.Lock()
servicePorts := resources.Network.ServicePorts[name]
if ports, ok := instanceBindings[id]; ok {
for guest, bindings := range ports {
status := servicePorts.Status[guest]
for _, binding := range bindings {
if contains(binding, status.Occupied) {
status.Occupied, status.Available = moveSpecificItem(binding, status.Occupied, status.Available)
} else {
log.WithFields(log.Fields{
"service": name,
"instance": id,
"guest": guest,
"host": binding,
}).Warnln("Cannot find port in occupied list")
}
}
servicePorts.Status[guest] = status
}
resources.Network.ServicePorts[name] = servicePorts
delete(instanceBindings, id)
log.WithFields(log.Fields{
"service": name,
"instance": id,
"ports": ports,
}).Debugln("Released ports of service instance")
} else {
log.WithFields(log.Fields{
"service": name,
"instance": id,
}).Warnln("Cannot find instance id in port map")
}
mutex_port.Unlock()
}
// /gru/v1/analytics/system
func GetAnalyticsSystem(w http.ResponseWriter, r *http.Request) {
analytics := analyzer.GetSystemAnalytics()
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(analytics); err != nil {
log.WithFields(log.Fields{
"status": "http response",
"request": "GetAnalyticsSystem",
"error": err,
}).Errorln("API Server")
}
}
func displayStatsOfServices(stats data.GruStats) {
for name, value := range stats.Metrics.Service {
service, _ := srv.GetServiceByName(name)
log.WithFields(log.Fields{
"pending:": len(service.Instances.Pending),
"running:": len(service.Instances.Running),
"stopped:": len(service.Instances.Stopped),
"paused:": len(service.Instances.Paused),
"cpu avg": fmt.Sprintf("%.2f", value.BaseMetrics[enum.METRIC_CPU_AVG.ToString()]),
"mem avg": fmt.Sprintf("%.2f", value.BaseMetrics[enum.METRIC_MEM_AVG.ToString()]),
}).Infoln("Stats computed: ", name)
}
}
// /gru/v1/actions
func GetInfoActions(w http.ResponseWriter, r *http.Request) {
actions := action.List()
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(actions); err != nil {
log.WithFields(log.Fields{
"status": "http response",
"request": "GetInfoPolicies",
"error": err,
}).Errorln("API Server")
}
}
// /gru/v1/shared/
func GetSharedData(w http.ResponseWriter, r *http.Request) {
info, _ := data.GetSharedLocal()
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(info); err != nil {
log.WithFields(log.Fields{
"status": "http response",
"request": "GetInfoNode",
"error": err,
}).Errorln("API Server")
}
}
func GetPortBindings(id string) (map[string][]string, error) {
info, err := docker.Client.InspectContainer(id)
if err != nil {
log.WithFields(log.Fields{
"id": id,
"err": err,
}).Errorln("Error inspecting instance")
}
portBindings := createPortBindings(info.HostConfig.PortBindings)
return portBindings, err
}
func ComputeMetricAnalytics(service string, metrics map[string]float64) map[string]float64 {
expressions := cfg.GetAnalyticExpr()
srvExprList := srv.GetServiceAnalyticsExprList(service)
srvConstraints := srv.GetServiceConstraints(service)
metricAnalytics := make(map[string]float64, len(expressions))
for _, expr := range srvExprList {
if curExpr, ok := expressions[expr]; ok {
log.WithField("expr", expr).Debugln("Evaluating expression")
toEval := buildExpression(curExpr, metrics, srvConstraints)
result, err := evaler.Eval(toEval)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"expr": toEval,
}).Warnln("Error evaluating expression")
metricAnalytics[expr] = 0.0
} else {
value := evaler.BigratToFloat(result)
value = math.Min(value, 1.0)
value = math.Max(value, 0.0)
metricAnalytics[expr] = value
}
log.WithFields(log.Fields{
"service": service,
"expr": expr,
"value": metricAnalytics[expr],
}).Debugln("Expression evaluated")
} else {
log.WithField("expr", expr).Errorln("Cannot compute expression: expression unknown")
}
}
return metricAnalytics
}
func GetDiscoveryPort(name string) string {
srv, err := GetServiceByName(name)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"service": name,
}).Debugln("Error getting discovery port")
return ""
}
return srv.DiscoveryPort
}
func (p *Remove) Run(config Action) error {
var err error
stopped := config.Instances.Stopped
if len(stopped) < 1 {
log.WithFields(log.Fields{
"service": config.Service,
"err": errNoContainerToRemove,
}).Errorln("Cannot remove container")
return errNoContainerToRemove
}
ch.SetRemovalNotification(true)
toRemove := stopped[0]
// Assumption: I remove only stopped containers; containers have no volume
err = container.Docker().Client.RemoveContainer(toRemove, false, false)
if err != nil {
log.WithFields(log.Fields{
"service": config.Service,
"instance": toRemove,
"err": err,
}).Errorln("Cannot remove container")
return err
}
waitForRemoval()
log.WithFields(log.Fields{
"service": config.Service,
"instance": toRemove,
}).Debugln("Removed container")
return nil
}
func addValue(entry logEntry) {
if entry.value < 0.0 {
log.WithFields(log.Fields{
"service": entry.service,
"metric": entry.metric,
"value": entry.value,
}).Warnln("Metric value < 0")
return
}
values := make([]float64, 0, 1)
values = append(values, entry.value)
mtr.UpdateUserMetric(entry.service, entry.metric, values)
}
func InitializeServiceAvailablePorts(name string, ports map[string]string) {
defer runtime.Gosched()
mutex_port.Lock()
servicePorts := resources.Network.ServicePorts[name]
for guest, host := range ports {
servicePorts.LastAssigned = make(map[string][]string)
servicePorts.Status = make(map[string]PortStatus)
hostRange, err := utils.GetCompleteRange(host)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"service": name,
"guest": guest,
"host": host,
}).Warnln("Cannot compute host port range for guest port")
}
status := PortStatus{
Available: hostRange,
Occupied: []string{},
}
servicePorts.Status[guest] = status
}
resources.Network.ServicePorts[name] = servicePorts
mutex_port.Unlock()
log.WithFields(log.Fields{
"service": name,
"ports": ports,
}).Debugln("Initialed service ports")
}
func UnregisterServiceInstance(name string, id string) {
discoveryConf := cfg.GetAgentDiscovery()
isntanceKey := discoveryConf.AppRoot + "/" + name + "/" + id
discovery.Delete(isntanceKey)
ch_stop, err := ch.GetInstanceChannel(id)
if err != nil {
log.WithFields(log.Fields{
"service": name,
"instance": id,
"err": err,
}).Errorln("Error getting instance stop channel")
return
}
ch_stop <- struct{}{}
ch.RemoveInstanceChannel(id)
}
func executeCommand(cmd Command) {
switch cmd.Name {
case "start":
startCommand(cmd)
case "stop":
stopCommand(cmd)
case "update":
updateCommand(cmd)
default:
log.Errorln("Unrecognized command name: ", cmd.Name)
}
log.WithFields(log.Fields{
"cmd": cmd.Name,
"target": cmd.Target,
}).Debugln("Executed command")
}
func TestLogstashFormatter(t *testing.T) {
assert := assert.New(t)
lf := LogstashFormatter{Type: "abc"}
fields := logrus.Fields{
"message": "def",
"level": "ijk",
"type": "lmn",
"one": 1,
"pi": 3.14,
"bool": true,
}
entry := logrus.WithFields(fields)
entry.Message = "msg"
entry.Level = logrus.InfoLevel
b, _ := lf.Format(entry)
var data map[string]interface{}
dec := json.NewDecoder(bytes.NewReader(b))
dec.UseNumber()
dec.Decode(&data)
// base fields
assert.Equal(json.Number("1"), data["@version"])
assert.NotEmpty(data["@timestamp"])
assert.Equal("abc", data["type"])
assert.Equal("msg", data["message"])
assert.Equal("info", data["level"])
// substituted fields
assert.Equal("def", data["fields.message"])
assert.Equal("ijk", data["fields.level"])
assert.Equal("lmn", data["fields.type"])
// formats
assert.Equal(json.Number("1"), data["one"])
assert.Equal(json.Number("3.14"), data["pi"])
assert.Equal(true, data["bool"])
}
func New(name string, conf map[string]interface{}) (metricService, error) {
metricServ = 0
for index, mtrc := range metricServices {
if mtrc.Name() == name {
err := mtrc.Initialize(conf)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"service": mtrc.Name(),
}).Errorln("Error initializing metric service")
return metricServices[metricServ], err
}
metricServ = index
log.WithField("service", name).Debugln("Initialized metric service")
return metricServices[metricServ], nil
}
}
return metricServices[metricServ], ErrNotSupported
}
func RegisterServiceInstanceId(name string, id string) {
var err error
discoveryConf := cfg.GetAgentDiscovery()
opt := discovery.Options{
"TTL": time.Duration(discoveryConf.TTL) * time.Second,
}
isntanceKey := discoveryConf.AppRoot + "/" + name + "/" + id
instanceValue := addressMap[id]
err = discovery.Set(isntanceKey, instanceValue, opt)
if err != nil {
log.WithFields(log.Fields{
"service": name,
"instance": id,
"address": instanceValue,
"err": err,
}).Errorln("Error registering service instance to discovery service")
}
}