func init() {
srv.SetMockServices()
setMockExpressions()
storage.New("internal")
constraints := map[string]float64{
"C1": 1.0,
"C2": 2.0,
}
for _, service := range srv.List() {
srv.SetServiceConstraints(service, constraints)
}
}
func initiailizeMonitoring() {
defer log.Infoln("Initializing autonomic monitoring")
ch_aut_err := chn.GetAutonomicErrChannel()
enableLogReading = cfg.GetAgentAutonomic().EnableLogReading
mtr.Initialize(srv.List())
// Start log reader if needed
if enableLogReading {
lgr.StartLogReader()
log.WithField("logreader", enableLogReading).Debugln("Log reading is enabled")
}
// Get the list of containers (running or not) to monitor
containers, err := container.Docker().Client.ListContainers(true, false, "")
if err != nil {
log.WithField("err", err).Debugln("Error monitoring containers")
ch_aut_err <- err
}
// Start the monitor for each configured service
for _, c := range containers {
info, _ := container.Docker().Client.InspectContainer(c.Id)
status := getContainerStatus(info)
service, err := srv.GetServiceByImage(c.Image)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"image": c.Image,
}).Warningln("Error monitoring service")
} else {
e := evt.Event{
Service: service.Name,
Image: c.Image,
Instance: c.Id,
Status: status,
}
evt.HandleCreateEvent(e)
evt.HanldeStartEvent(e)
mtr.AddInstance(c.Id)
if _, ok := instBuffer[c.Id]; !ok {
instBuffer[c.Id] = instanceMetricBuffer{
cpuInst: utils.BuildBuffer(c_B_SIZE),
cpuSys: utils.BuildBuffer(c_B_SIZE),
}
}
container.Docker().Client.StartMonitorStats(c.Id, statCallBack, ch_mnt_stats_err)
if status == enum.PENDING && enableLogReading {
startMonitorLog(c.Id)
}
}
}
}
func Run() data.GruStats {
log.WithField("status", "init").Debugln("Gru Monitor")
defer log.WithField("status", "done").Debugln("Gru Monitor")
services := srv.List()
updateNodeResources()
updateRunningInstances(services, c_MTR_THR)
updateSystemInstances(services)
metrics := mtr.GetMetricsStats()
events := evt.GetEventsStats()
stats.Metrics = metrics
stats.Events = events
data.SaveStats(stats)
displayStatsOfServices(stats)
return stats
}
func newMetrics() GruMetric {
metricsEmpty := GruMetric{Service: make(map[string]ServiceMetric)}
metricDataEmpty := data.MetricData{
BaseMetrics: map[string]float64{
enum.METRIC_CPU_AVG.ToString(): 0.0,
enum.METRIC_MEM_AVG.ToString(): 0.0,
},
UserMetrics: map[string]float64{},
}
analyticsDataEmpty := data.AnalyticData{
BaseAnalytics: map[string]float64{
enum.METRIC_CPU_AVG.ToString(): 0.0,
enum.METRIC_MEM_AVG.ToString(): 0.0,
},
UserAnalytics: map[string]float64{},
}
sharedDataEmpty := data.SharedData{
BaseShared: map[string]float64{
enum.METRIC_CPU_AVG.ToString(): 0.0,
enum.METRIC_MEM_AVG.ToString(): 0.0,
},
UserShared: map[string]float64{},
}
metricsEmpty.Node = NodeMetrics{
Stats: metricDataEmpty,
}
for _, name := range service.List() {
metricsEmpty.Service[name] = ServiceMetric{
Stats: metricDataEmpty,
Analytics: analyticsDataEmpty,
Shared: sharedDataEmpty,
}
}
policyEmpty := PolicyMetric{Name: "noaction", Weight: 1.0}
metricsEmpty.Policy = policyEmpty
return metricsEmpty
}
func TestUpdateSystemInstances(t *testing.T) {
defer resetMockServices()
list := srv.List()
updateSystemInstances(list)
srv1, _ := srv.GetServiceByName("service1")
srv2, _ := srv.GetServiceByName("service2")
tot_all := len(srv1.Instances.All) + len(srv2.Instances.All)
tot_pen := len(srv1.Instances.Pending) + len(srv2.Instances.Pending)
tot_run := len(srv1.Instances.Running) + len(srv2.Instances.Running)
tot_stop := len(srv1.Instances.Stopped) + len(srv2.Instances.Stopped)
tot_pause := len(srv1.Instances.Paused) + len(srv2.Instances.Paused)
instances := cfg.GetNodeInstances()
assert.Len(t, instances.All, tot_all)
assert.Len(t, instances.Pending, tot_pen)
assert.Len(t, instances.Running, tot_run)
assert.Len(t, instances.Stopped, tot_stop)
assert.Len(t, instances.Paused, tot_pause)
}
func TestResetMetrics(t *testing.T) {
id := "id1"
defer delete(instancesMetrics, id)
instancesMetrics[id] = Metric{
BaseMetrics: make(map[string][]float64),
}
instancesMetrics[id].BaseMetrics[enum.METRIC_CPU_INST.ToString()] = []float64{10000, 20000, 30000, 40000, 50000, 60000}
instancesMetrics[id].BaseMetrics[enum.METRIC_CPU_SYS.ToString()] = []float64{1000000, 1100000, 1200000, 1300000, 1400000, 1500000}
servicesMetrics["service1"].UserMetrics["response_time"] = []float64{1000, 2000, 3000, 4000, 5000}
servicesMetrics["service2"].UserMetrics["response_time"] = []float64{1000, 2000, 3000, 4000, 5000}
resetMetrics()
assert.Len(t, servicesMetrics, len(srv.List()))
assert.Empty(t, servicesMetrics["service1"].UserMetrics["response_time"])
assert.Empty(t, servicesMetrics["service2"].UserMetrics["response_time"])
assert.Empty(t, servicesMetrics["service3"].UserMetrics["response_time"])
assert.Len(t, instancesMetrics, 1)
assert.Empty(t, instancesMetrics[id].BaseMetrics[enum.METRIC_CPU_INST.ToString()])
assert.Empty(t, instancesMetrics[id].BaseMetrics[enum.METRIC_CPU_SYS.ToString()])
}
func init() {
res.CreateMockResources(1, "1G", 0, "0G")
resetMockServices()
Initialize(srv.List())
}
func initializeNetworkResources() {
for _, name := range service.List() {
srv, _ := service.GetServiceByName(name)
InitializeServiceAvailablePorts(name, srv.Docker.Ports)
}
}
func MergeShared(toMerge []Shared) (Shared, error) {
if len(toMerge) < 1 {
return Shared{}, errors.New("No shared data to merge")
}
if len(toMerge) == 1 {
return toMerge[0], nil
}
merged := Shared{
Service: make(map[string]ServiceShared),
}
for _, name := range srv.List() {
srvMerged := ServiceShared{}
baseValues := make(map[string][]float64)
userValues := make(map[string][]float64)
for _, data := range toMerge {
if data.Service[name].Active {
srvMerged.Active = true
for analytics, value := range data.Service[name].Data.BaseShared {
baseValues[analytics] = append(baseValues[analytics], value)
}
for analytics, value := range data.Service[name].Data.UserShared {
userValues[analytics] = append(userValues[analytics], value)
}
}
}
baseMerged := make(map[string]float64, len(baseValues))
for analytics, values := range baseValues {
baseMerged[analytics] = utils.Mean(values)
}
userMerged := make(map[string]float64, len(userValues))
for analytics, values := range userValues {
userMerged[analytics] = utils.Mean(values)
}
srvMerged.Data.BaseShared = baseMerged
srvMerged.Data.UserShared = userMerged
merged.Service[name] = srvMerged
}
sysMerged := SystemShared{}
baseValues := make(map[string][]float64)
userValues := make(map[string][]float64)
for _, data := range toMerge {
for analytics, value := range data.System.Data.BaseShared {
baseValues[analytics] = append(baseValues[analytics], value)
}
for analytics, value := range data.System.Data.UserShared {
userValues[analytics] = append(userValues[analytics], value)
}
sysMerged.ActiveServices = checkAndAppend(sysMerged.ActiveServices, data.System.ActiveServices)
}
baseMerged := make(map[string]float64, len(baseValues))
for analytics, values := range baseValues {
baseMerged[analytics] = utils.Mean(values)
}
userMerged := make(map[string]float64, len(userValues))
for analytics, values := range userValues {
userMerged[analytics] = utils.Mean(values)
}
sysMerged.Data.BaseShared = baseMerged
sysMerged.Data.UserShared = userMerged
merged.System = sysMerged
return merged, nil
// var loadAvg float64
// var cpuAvg float64
// var memAvg float64
// var resourcesAvg float64
// for _, name := range service.List() {
// loadAvg = 0.0
// cpuAvg = 0.0
// memAvg = 0.0
// resourcesAvg = 0.0
// counter := 0.0
// for _, info := range toMerge {
// if srv, ok := info.Service[name]; ok {
// if srv.Active {
// loadAvg += srv.Load
// cpuAvg += srv.Cpu
// memAvg += srv.Memory
// resourcesAvg += srv.Resources
// counter++
// }
// }
// }
// if counter > 0 {
// loadAvg /= counter
// cpuAvg /= counter
// memAvg /= counter
// resourcesAvg /= counter
// mergedService := ServiceShared{
// Load: loadAvg,
// Cpu: cpuAvg,
// Memory: memAvg,
// Resources: resourcesAvg,
// Active: true,
// }
// merged.Service[name] = mergedService
// }
// }
// cpuAvg = 0.0
// memAvg = 0.0
// resourcesAvg = 0.0
// healthAvg := 0.0
// activeServices := []string{}
// for _, info := range toMerge {
// cpuAvg += info.System.Cpu
// memAvg += info.System.Memory
// healthAvg += info.System.Health
// activeServices = checkAndAppend(activeServices, info.System.ActiveServices)
// }
// lenght := float64(len(toMerge))
// cpuAvg /= lenght
// memAvg /= lenght
// healthAvg /= lenght
// mergedSystem := SystemShared{
// Cpu: cpuAvg,
// Memory: memAvg,
// Health: healthAvg,
// ActiveServices: activeServices,
// }
// merged.System = mergedSystem
}
func updateMetrics() {
var err error
metrics = newMetrics()
metrics.Node.UUID = cfg.GetNodeConfig().UUID
metrics.Node.Name = cfg.GetNodeConfig().Name
metrics.Node.Resources.CPU.Total = res.GetResources().CPU.Total
metrics.Node.Resources.CPU.Availabe = res.GetResources().CPU.Total - res.GetResources().CPU.Used
localShared, err := data.GetSharedLocal()
if err != nil {
log.WithField("err", err).Warnln("Cannot update node active metric")
} else {
metrics.Node.ActiveServices = len(localShared.System.ActiveServices)
}
for _, name := range service.List() {
srv, _ := service.GetServiceByName(name)
srv_metrics := metrics.Service[name]
srv_metrics.Name = name
srv_metrics.Image = srv.Image
srv_metrics.Type = srv.Type
srv_metrics.Instances.All = len(srv.Instances.All)
srv_metrics.Instances.Pending = len(srv.Instances.Pending)
srv_metrics.Instances.Running = len(srv.Instances.Running)
srv_metrics.Instances.Paused = len(srv.Instances.Paused)
srv_metrics.Instances.Stopped = len(srv.Instances.Stopped)
stats, err := data.GetStats()
if err != nil {
log.WithFields(log.Fields{
"err": err,
"service": name,
}).Warnln("Cannot update stats metrics")
} else {
if srv_stats, ok := stats.Metrics.Service[name]; ok {
srv_metrics.Stats = srv_stats
} else {
log.Warnln("Cannot find stats metrics for service ", name)
}
metrics.Node.Stats = stats.Metrics.System
}
analytics, err := data.GetAnalytics()
if err != nil {
log.WithFields(log.Fields{
"err": err,
"service": name,
}).Warnln("Cannot update analytics metrics")
} else {
if srv_analytisc, ok := analytics.Service[name]; ok {
srv_metrics.Analytics = srv_analytisc
} else {
log.Debugln("Cannot find analytics metrics for service ", name)
}
}
shared, err := data.GetSharedCluster()
if err != nil {
log.WithFields(log.Fields{
"err": err,
"service": name,
}).Warnln("Cannot update shared data metrics")
} else {
if srv_shared, ok := shared.Service[name]; ok {
srv_metrics.Shared = srv_shared.Data
}
}
metrics.Service[name] = srv_metrics
}
plc, err := data.GetPolicy()
if err != nil {
log.WithField("err", err).Warnln("Cannot update plans metrics")
} else {
metrics.Policy.Name = plc.Name
metrics.Policy.Weight = plc.Weight
}
}