Example #1
0
func init() {
	srv.SetMockServices()
	setMockExpressions()
	storage.New("internal")
	constraints := map[string]float64{
		"C1": 1.0,
		"C2": 2.0,
	}
	for _, service := range srv.List() {
		srv.SetServiceConstraints(service, constraints)
	}
}
Example #2
0
func initiailizeMonitoring() {
	defer log.Infoln("Initializing autonomic monitoring")
	ch_aut_err := chn.GetAutonomicErrChannel()
	enableLogReading = cfg.GetAgentAutonomic().EnableLogReading
	mtr.Initialize(srv.List())

	// Start log reader if needed
	if enableLogReading {
		lgr.StartLogReader()
		log.WithField("logreader", enableLogReading).Debugln("Log reading is enabled")
	}

	// Get the list of containers (running or not) to monitor
	containers, err := container.Docker().Client.ListContainers(true, false, "")
	if err != nil {
		log.WithField("err", err).Debugln("Error monitoring containers")
		ch_aut_err <- err
	}

	// Start the monitor for each configured service
	for _, c := range containers {
		info, _ := container.Docker().Client.InspectContainer(c.Id)
		status := getContainerStatus(info)
		service, err := srv.GetServiceByImage(c.Image)
		if err != nil {
			log.WithFields(log.Fields{
				"err":   err,
				"image": c.Image,
			}).Warningln("Error monitoring service")
		} else {
			e := evt.Event{
				Service:  service.Name,
				Image:    c.Image,
				Instance: c.Id,
				Status:   status,
			}

			evt.HandleCreateEvent(e)
			evt.HanldeStartEvent(e)
			mtr.AddInstance(c.Id)
			if _, ok := instBuffer[c.Id]; !ok {
				instBuffer[c.Id] = instanceMetricBuffer{
					cpuInst: utils.BuildBuffer(c_B_SIZE),
					cpuSys:  utils.BuildBuffer(c_B_SIZE),
				}
			}
			container.Docker().Client.StartMonitorStats(c.Id, statCallBack, ch_mnt_stats_err)
			if status == enum.PENDING && enableLogReading {
				startMonitorLog(c.Id)
			}
		}
	}
}
Example #3
0
func Run() data.GruStats {
	log.WithField("status", "init").Debugln("Gru Monitor")
	defer log.WithField("status", "done").Debugln("Gru Monitor")

	services := srv.List()
	updateNodeResources()
	updateRunningInstances(services, c_MTR_THR)
	updateSystemInstances(services)
	metrics := mtr.GetMetricsStats()
	events := evt.GetEventsStats()
	stats.Metrics = metrics
	stats.Events = events
	data.SaveStats(stats)
	displayStatsOfServices(stats)
	return stats
}
Example #4
0
func newMetrics() GruMetric {
	metricsEmpty := GruMetric{Service: make(map[string]ServiceMetric)}
	metricDataEmpty := data.MetricData{
		BaseMetrics: map[string]float64{
			enum.METRIC_CPU_AVG.ToString(): 0.0,
			enum.METRIC_MEM_AVG.ToString(): 0.0,
		},
		UserMetrics: map[string]float64{},
	}
	analyticsDataEmpty := data.AnalyticData{
		BaseAnalytics: map[string]float64{
			enum.METRIC_CPU_AVG.ToString(): 0.0,
			enum.METRIC_MEM_AVG.ToString(): 0.0,
		},
		UserAnalytics: map[string]float64{},
	}
	sharedDataEmpty := data.SharedData{
		BaseShared: map[string]float64{
			enum.METRIC_CPU_AVG.ToString(): 0.0,
			enum.METRIC_MEM_AVG.ToString(): 0.0,
		},
		UserShared: map[string]float64{},
	}

	metricsEmpty.Node = NodeMetrics{
		Stats: metricDataEmpty,
	}

	for _, name := range service.List() {
		metricsEmpty.Service[name] = ServiceMetric{
			Stats:     metricDataEmpty,
			Analytics: analyticsDataEmpty,
			Shared:    sharedDataEmpty,
		}
	}

	policyEmpty := PolicyMetric{Name: "noaction", Weight: 1.0}
	metricsEmpty.Policy = policyEmpty

	return metricsEmpty
}
Example #5
0
func TestUpdateSystemInstances(t *testing.T) {
	defer resetMockServices()
	list := srv.List()

	updateSystemInstances(list)

	srv1, _ := srv.GetServiceByName("service1")
	srv2, _ := srv.GetServiceByName("service2")
	tot_all := len(srv1.Instances.All) + len(srv2.Instances.All)
	tot_pen := len(srv1.Instances.Pending) + len(srv2.Instances.Pending)
	tot_run := len(srv1.Instances.Running) + len(srv2.Instances.Running)
	tot_stop := len(srv1.Instances.Stopped) + len(srv2.Instances.Stopped)
	tot_pause := len(srv1.Instances.Paused) + len(srv2.Instances.Paused)

	instances := cfg.GetNodeInstances()
	assert.Len(t, instances.All, tot_all)
	assert.Len(t, instances.Pending, tot_pen)
	assert.Len(t, instances.Running, tot_run)
	assert.Len(t, instances.Stopped, tot_stop)
	assert.Len(t, instances.Paused, tot_pause)
}
Example #6
0
func TestResetMetrics(t *testing.T) {
	id := "id1"
	defer delete(instancesMetrics, id)

	instancesMetrics[id] = Metric{
		BaseMetrics: make(map[string][]float64),
	}
	instancesMetrics[id].BaseMetrics[enum.METRIC_CPU_INST.ToString()] = []float64{10000, 20000, 30000, 40000, 50000, 60000}
	instancesMetrics[id].BaseMetrics[enum.METRIC_CPU_SYS.ToString()] = []float64{1000000, 1100000, 1200000, 1300000, 1400000, 1500000}
	servicesMetrics["service1"].UserMetrics["response_time"] = []float64{1000, 2000, 3000, 4000, 5000}
	servicesMetrics["service2"].UserMetrics["response_time"] = []float64{1000, 2000, 3000, 4000, 5000}

	resetMetrics()
	assert.Len(t, servicesMetrics, len(srv.List()))
	assert.Empty(t, servicesMetrics["service1"].UserMetrics["response_time"])
	assert.Empty(t, servicesMetrics["service2"].UserMetrics["response_time"])
	assert.Empty(t, servicesMetrics["service3"].UserMetrics["response_time"])
	assert.Len(t, instancesMetrics, 1)
	assert.Empty(t, instancesMetrics[id].BaseMetrics[enum.METRIC_CPU_INST.ToString()])
	assert.Empty(t, instancesMetrics[id].BaseMetrics[enum.METRIC_CPU_SYS.ToString()])

}
Example #7
0
func init() {
	res.CreateMockResources(1, "1G", 0, "0G")
	resetMockServices()
	Initialize(srv.List())
}
Example #8
0
func initializeNetworkResources() {
	for _, name := range service.List() {
		srv, _ := service.GetServiceByName(name)
		InitializeServiceAvailablePorts(name, srv.Docker.Ports)
	}
}
Example #9
0
func MergeShared(toMerge []Shared) (Shared, error) {
	if len(toMerge) < 1 {
		return Shared{}, errors.New("No shared data to merge")
	}

	if len(toMerge) == 1 {
		return toMerge[0], nil
	}

	merged := Shared{
		Service: make(map[string]ServiceShared),
	}

	for _, name := range srv.List() {
		srvMerged := ServiceShared{}
		baseValues := make(map[string][]float64)
		userValues := make(map[string][]float64)
		for _, data := range toMerge {
			if data.Service[name].Active {
				srvMerged.Active = true

				for analytics, value := range data.Service[name].Data.BaseShared {
					baseValues[analytics] = append(baseValues[analytics], value)
				}

				for analytics, value := range data.Service[name].Data.UserShared {
					userValues[analytics] = append(userValues[analytics], value)
				}
			}
		}

		baseMerged := make(map[string]float64, len(baseValues))
		for analytics, values := range baseValues {
			baseMerged[analytics] = utils.Mean(values)
		}

		userMerged := make(map[string]float64, len(userValues))
		for analytics, values := range userValues {
			userMerged[analytics] = utils.Mean(values)
		}

		srvMerged.Data.BaseShared = baseMerged
		srvMerged.Data.UserShared = userMerged
		merged.Service[name] = srvMerged
	}

	sysMerged := SystemShared{}
	baseValues := make(map[string][]float64)
	userValues := make(map[string][]float64)
	for _, data := range toMerge {

		for analytics, value := range data.System.Data.BaseShared {
			baseValues[analytics] = append(baseValues[analytics], value)
		}

		for analytics, value := range data.System.Data.UserShared {
			userValues[analytics] = append(userValues[analytics], value)
		}

		sysMerged.ActiveServices = checkAndAppend(sysMerged.ActiveServices, data.System.ActiveServices)
	}

	baseMerged := make(map[string]float64, len(baseValues))
	for analytics, values := range baseValues {
		baseMerged[analytics] = utils.Mean(values)
	}

	userMerged := make(map[string]float64, len(userValues))
	for analytics, values := range userValues {
		userMerged[analytics] = utils.Mean(values)
	}

	sysMerged.Data.BaseShared = baseMerged
	sysMerged.Data.UserShared = userMerged
	merged.System = sysMerged

	return merged, nil

	// var loadAvg float64
	// var cpuAvg float64
	// var memAvg float64
	// var resourcesAvg float64

	// for _, name := range service.List() {
	// 	loadAvg = 0.0
	// 	cpuAvg = 0.0
	// 	memAvg = 0.0
	// 	resourcesAvg = 0.0
	// 	counter := 0.0
	// 	for _, info := range toMerge {
	// 		if srv, ok := info.Service[name]; ok {
	// 			if srv.Active {
	// 				loadAvg += srv.Load
	// 				cpuAvg += srv.Cpu
	// 				memAvg += srv.Memory
	// 				resourcesAvg += srv.Resources

	// 				counter++
	// 			}
	// 		}
	// 	}

	// 	if counter > 0 {
	// 		loadAvg /= counter
	// 		cpuAvg /= counter
	// 		memAvg /= counter
	// 		resourcesAvg /= counter

	// 		mergedService := ServiceShared{
	// 			Load:      loadAvg,
	// 			Cpu:       cpuAvg,
	// 			Memory:    memAvg,
	// 			Resources: resourcesAvg,
	// 			Active:    true,
	// 		}

	// 		merged.Service[name] = mergedService
	// 	}
	// }

	// cpuAvg = 0.0
	// memAvg = 0.0
	// resourcesAvg = 0.0
	// healthAvg := 0.0
	// activeServices := []string{}
	// for _, info := range toMerge {
	// 	cpuAvg += info.System.Cpu
	// 	memAvg += info.System.Memory
	// 	healthAvg += info.System.Health
	// 	activeServices = checkAndAppend(activeServices, info.System.ActiveServices)
	// }

	// lenght := float64(len(toMerge))
	// cpuAvg /= lenght
	// memAvg /= lenght
	// healthAvg /= lenght

	// mergedSystem := SystemShared{
	// 	Cpu:            cpuAvg,
	// 	Memory:         memAvg,
	// 	Health:         healthAvg,
	// 	ActiveServices: activeServices,
	// }

	// merged.System = mergedSystem
}
Example #10
0
func updateMetrics() {
	var err error
	metrics = newMetrics()
	metrics.Node.UUID = cfg.GetNodeConfig().UUID
	metrics.Node.Name = cfg.GetNodeConfig().Name
	metrics.Node.Resources.CPU.Total = res.GetResources().CPU.Total
	metrics.Node.Resources.CPU.Availabe = res.GetResources().CPU.Total - res.GetResources().CPU.Used
	localShared, err := data.GetSharedLocal()
	if err != nil {
		log.WithField("err", err).Warnln("Cannot update node active metric")
	} else {
		metrics.Node.ActiveServices = len(localShared.System.ActiveServices)
	}

	for _, name := range service.List() {
		srv, _ := service.GetServiceByName(name)
		srv_metrics := metrics.Service[name]
		srv_metrics.Name = name
		srv_metrics.Image = srv.Image
		srv_metrics.Type = srv.Type

		srv_metrics.Instances.All = len(srv.Instances.All)
		srv_metrics.Instances.Pending = len(srv.Instances.Pending)
		srv_metrics.Instances.Running = len(srv.Instances.Running)
		srv_metrics.Instances.Paused = len(srv.Instances.Paused)
		srv_metrics.Instances.Stopped = len(srv.Instances.Stopped)

		stats, err := data.GetStats()
		if err != nil {
			log.WithFields(log.Fields{
				"err":     err,
				"service": name,
			}).Warnln("Cannot update stats metrics")
		} else {
			if srv_stats, ok := stats.Metrics.Service[name]; ok {
				srv_metrics.Stats = srv_stats
			} else {
				log.Warnln("Cannot find stats metrics for service ", name)
			}

			metrics.Node.Stats = stats.Metrics.System
		}

		analytics, err := data.GetAnalytics()
		if err != nil {
			log.WithFields(log.Fields{
				"err":     err,
				"service": name,
			}).Warnln("Cannot update analytics metrics")
		} else {
			if srv_analytisc, ok := analytics.Service[name]; ok {
				srv_metrics.Analytics = srv_analytisc
			} else {
				log.Debugln("Cannot find analytics metrics for service ", name)
			}
		}

		shared, err := data.GetSharedCluster()
		if err != nil {
			log.WithFields(log.Fields{
				"err":     err,
				"service": name,
			}).Warnln("Cannot update shared data metrics")
		} else {
			if srv_shared, ok := shared.Service[name]; ok {
				srv_metrics.Shared = srv_shared.Data
			}
		}

		metrics.Service[name] = srv_metrics
	}

	plc, err := data.GetPolicy()
	if err != nil {
		log.WithField("err", err).Warnln("Cannot update plans metrics")
	} else {
		metrics.Policy.Name = plc.Name
		metrics.Policy.Weight = plc.Weight
	}

}