func buildExpression(expr cfg.AnalyticExpr, metrics map[string]float64, constraints map[string]float64) string {
toBuild := expr.Expr
for _, metric := range expr.Metrics {
if value, ok := metrics[metric]; ok {
toBuild = strings.Replace(toBuild, metric, strconv.FormatFloat(value, 'f', -1, 64), -1)
} else {
log.WithField("metric", metric).Debugln("Cannot build expression: metric unknown")
return "noexp"
}
}
for _, constraint := range expr.Constraints {
if value, ok := constraints[constraint]; ok {
toBuild = strings.Replace(toBuild, constraint, strconv.FormatFloat(value, 'f', -1, 64), -1)
} else {
log.WithField("constraint", constraint).Debugln("Cannot build expression: constraint unknown")
return "noexp"
}
}
return toBuild
}
func getFriendsData(friends map[string]string) ([]data.Shared, error) {
var err error
friendsData := make([]data.Shared, 0, len(friends))
for friend, address := range friends {
friendRoute := address + c_ROUTE_SHARED
friendData, err := network.DoRequest("GET", friendRoute, nil)
if err != nil {
log.WithField("address", address).Debugln("Error retrieving friend stats")
}
err = storage.StoreData(friend, friendData, enum.SHARED)
if err != nil {
log.WithField("err", err).Debugln("Error storing friends data")
}
sharedData, err := data.ByteToShared(friendData)
if err != nil {
log.WithField("address", address).Debugln("Friend data not stored")
} else {
friendsData = append(friendsData, sharedData)
}
}
return friendsData, err
}
func (p *Stop) Run(config Action) error {
var err error
var toStop string
running := config.Instances.Running
if len(running) < 1 {
log.WithField("err", ErrNoContainerToStop).Errorln("Cannot stop running container. Trying with pending ones...")
pending := config.Instances.Pending
if len(pending) < 1 {
log.WithField("err", ErrNoContainerToStop).Errorln("Cannot stop pending container")
return ErrNoContainerToStop
}
toStop = pending[0]
} else {
toStop = running[0]
}
err = container.Docker().Client.StopContainer(toStop, config.Parameters.StopTimeout)
if err != nil {
log.WithField("err", err).Errorln("Cannot stop container ", toStop)
return err
}
return nil
}
func readCommand(r *http.Request) (Command, error) {
var err error
var cmd Command
body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
if err != nil {
log.WithField("err", err).Errorln("Error reading command body")
return Command{}, err
}
if err = r.Body.Close(); err != nil {
log.WithField("err", err).Errorln("Error closing command body")
return Command{}, err
}
if err = json.Unmarshal(body, &cmd); err != nil {
log.WithField("err", err).Errorln("Error unmarshaling command body")
return Command{}, err
}
cmd.Timestamp = time.Now()
log.WithFields(log.Fields{
"name": cmd.Name,
"target": cmd.Target,
"timestamp": cmd.Timestamp,
}).Debugln("Received command")
return cmd, nil
}
func RemoveServiceInstance(name string, instance string) error {
service, err := getServiceBy("name", name)
if err != nil {
log.WithField("service", name).Errorln("Cannot remove service instance: service unknown")
return err
}
instIndexStop, err := findIdIndex(instance, service.Instances.Stopped)
if err != nil {
log.WithField("instance", instance).Errorln("Cannot remove service instance: instance unknown")
return err
}
instIndexAll, err := findIdIndex(instance, service.Instances.All)
if err != nil {
log.WithField("instance", instance).Errorln("Cannot remove service instance: instance unknown")
return err
}
service.Instances.Stopped = append(service.Instances.Stopped[:instIndexStop],
service.Instances.Stopped[instIndexStop+1:]...)
service.Instances.All = append(service.Instances.All[:instIndexAll],
service.Instances.All[instIndexAll+1:]...)
return nil
}
func initializeNetwork(address string, port string) {
err := network.InitializeNetwork(address, port)
if err != nil {
log.WithField("err", err).Fatalln("Error initializing the network")
}
log.WithField(network.Config().IpAddress+":"+network.Config().Port, "ok").Infoln("Network initialized")
}
func updateFriendsData(nFriends int) error {
var err error
log.Debugln("Updating friends data")
err = clearFriendsData()
if err != nil {
return err
}
peers := getAllPeers()
log.WithField("peers", len(peers)).Debugln("Number of peers")
if len(peers) == 0 {
return ErrNoPeers
}
friends, err := chooseRandomFriends(peers, nFriends)
if err != nil {
return err
}
log.WithField("friends", friends).Debugln("Friends to connect with")
friendsData, err := getFriendsData(friends)
if err != nil {
return err
}
clusterData, err := mergeSharedData(friendsData)
if err != nil {
return err
}
data.SaveSharedCluster(clusterData)
return nil
}
func startService(name string) {
log.WithField("name", name).Debugln("Starting service")
toStart, err := service.GetServiceByName(name)
if err != nil {
log.WithField("name", name).Debugln("Error starting service")
}
ch.SendActionStartMessage(toStart)
}
// TODO needs to make more "generic" the container engine package
func initializeContainerEngine() {
daemonUrl := getDaemonUrl()
log.WithField("daemonUrl", daemonUrl).Debugln("Container engine initialization")
err := container.Connect(daemonUrl, cfg.GetAgentDocker().DaemonTimeout)
if err != nil {
log.WithField("err", err).Fatalln("Error initializing container engine")
}
log.WithField("docker", "ok").Infoln("Container engine initialized")
}
func checkConfiguration(name string, conf *cfg.ServiceDocker) {
if conf.Memory == "" {
log.WithField("service", name).Warnln("Memory limit not set. Service will use all the memory available")
}
if conf.CpusetCpus == "" {
log.WithField("service", name).Warnln("Cores not assigned. Service will use all the cores")
}
}
func SetPlannerStrategy(strategyName string) {
strtg, err := strategy.New(strategyName)
if err != nil {
log.WithField("err", err).Errorln("Strategy cannot be set")
// If error use default one
strtg, err = strategy.New("dummy")
}
currentStrategy = strtg
log.WithField("strategy", strtg.Name()).Infoln("Strategy initialized")
}
func initiailizeMonitoring() {
defer log.Infoln("Initializing autonomic monitoring")
ch_aut_err := chn.GetAutonomicErrChannel()
enableLogReading = cfg.GetAgentAutonomic().EnableLogReading
mtr.Initialize(srv.List())
// Start log reader if needed
if enableLogReading {
lgr.StartLogReader()
log.WithField("logreader", enableLogReading).Debugln("Log reading is enabled")
}
// Get the list of containers (running or not) to monitor
containers, err := container.Docker().Client.ListContainers(true, false, "")
if err != nil {
log.WithField("err", err).Debugln("Error monitoring containers")
ch_aut_err <- err
}
// Start the monitor for each configured service
for _, c := range containers {
info, _ := container.Docker().Client.InspectContainer(c.Id)
status := getContainerStatus(info)
service, err := srv.GetServiceByImage(c.Image)
if err != nil {
log.WithFields(log.Fields{
"err": err,
"image": c.Image,
}).Warningln("Error monitoring service")
} else {
e := evt.Event{
Service: service.Name,
Image: c.Image,
Instance: c.Id,
Status: status,
}
evt.HandleCreateEvent(e)
evt.HanldeStartEvent(e)
mtr.AddInstance(c.Id)
if _, ok := instBuffer[c.Id]; !ok {
instBuffer[c.Id] = instanceMetricBuffer{
cpuInst: utils.BuildBuffer(c_B_SIZE),
cpuSys: utils.BuildBuffer(c_B_SIZE),
}
}
container.Docker().Client.StartMonitorStats(c.Id, statCallBack, ch_mnt_stats_err)
if status == enum.PENDING && enableLogReading {
startMonitorLog(c.Id)
}
}
}
}
func ComputeUsedResources() {
var err error
_, err = ComputeUsedCpus()
if err != nil {
log.WithField("err", err).Errorln("Error computing used CPU")
}
_, err = ComputeUsedMemory()
if err != nil {
log.WithField("err", err).Errorln("Error computing used Memory")
}
}
func stopService(name string) {
log.WithField("name", name).Debugln("Stopping service")
toStop, err := service.GetServiceByName(name)
if err != nil {
log.WithField("name", name).Debugln("Error stopping service")
}
if len(toStop.Instances.All) < 1 {
log.WithField("service", name).Debugln("No active instance to stop")
return
}
ch.SendActionStopMessage(toStop)
}
func computeMetrics() data.MetricStats {
metrics := data.MetricStats{}
instMetrics := computeInstancesMetrics()
log.WithField("instMetrics", instMetrics).Debugln("Computed instances metrics")
servMetrics := computeServicesMetrics(instMetrics)
log.WithField("servMetrics", servMetrics).Debugln("Computed service metrics")
sysMetrics := computeSysMetrics(instMetrics)
log.WithField("sysMetrics", sysMetrics).Debugln("Computed system metrics")
metrics.Instance = instMetrics
metrics.Service = servMetrics
metrics.System = sysMetrics
return metrics
}
func ReadNodeActive(remote string) bool {
activePath := remote + "/active"
resp, err := discovery.Get(activePath, discovery.Options{})
if err != nil {
log.WithField("err", err).Errorln("Error reading node active")
return false
}
active, err := strconv.ParseBool(resp[activePath])
if err != nil {
log.WithField("err", err).Errorln("Error parsing node active")
return false
}
return active
}
func Run() data.GruStats {
log.WithField("status", "init").Debugln("Gru Monitor")
defer log.WithField("status", "done").Debugln("Gru Monitor")
services := srv.List()
updateNodeResources()
updateRunningInstances(services, c_MTR_THR)
updateSystemInstances(services)
metrics := mtr.GetMetricsStats()
events := evt.GetEventsStats()
stats.Metrics = metrics
stats.Events = events
data.SaveStats(stats)
displayStatsOfServices(stats)
return stats
}
func Run(chosenPolicy *data.Policy) {
log.WithField("status", "init").Debugln("Gru Executor")
defer log.WithField("status", "done").Debugln("Gru Executor")
if chosenPolicy == nil {
log.Warnln("No policy to execute")
return
}
for _, target := range chosenPolicy.Targets {
actions := chosenPolicy.Actions[target]
srv := getTargetService(target)
executeActions(srv, actions)
}
}
func ReadNodes(remote string) []Node {
log.WithField("remote", remote).Debugln("Reading nodes from remote")
resp, err := discovery.Get(remote, discovery.Options{})
if err != nil {
log.WithField("err", err).Errorln("Error reading nodes from ", remote)
return []Node{}
}
nodes := []Node{}
for nodePath, _ := range resp {
n := ReadNode(nodePath)
nodes = append(nodes, n)
}
return nodes
}
func WriteNodeResources(remote string, data NodeResources) {
resourcesPath := remote + "/resources"
err := writeData(resourcesPath, data)
if err != nil {
log.WithField("err", err).Errorln("Error writing node resources")
}
}
func ReadNodeResources(remote string, resources *NodeResources) {
resourcesPath := remote + "/resources"
err := readData(resourcesPath, resources)
if err != nil {
log.WithField("err", err).Errorln("Error reading node resources")
}
}
func mergeSharedData(friendsData []data.Shared) (data.Shared, error) {
sharedStored, err := data.GetSharedCluster()
if err != nil {
log.WithField("err", err).Debugln("Cannot get stored shared data")
return data.MergeShared(friendsData)
}
toMerge := append(friendsData, sharedStored)
sharedCluster, err := data.MergeShared(toMerge)
if err != nil {
log.WithField("err", err).Debugln("Cannot merge friends data")
return data.Shared{}, err
}
return sharedCluster, nil
}
func WriteNodeConstraints(remote string, data NodeConstraints) {
constraintsPath := remote + "/constraints"
err := writeData(constraintsPath, data)
if err != nil {
log.WithField("err", err).Errorln("Error writing node constraints")
}
}
func WriteNodeConfig(remote string, data NodeConfig) {
configPath := remote + "/config"
err := writeData(configPath, data)
if err != nil {
log.WithField("err", err).Errorln("Error writing node configuration")
}
}
func ReadNodeConstraints(remote string, constraints *NodeConstraints) {
constraintsPath := remote + "/constraints"
err := readData(constraintsPath, constraints)
if err != nil {
log.WithField("err", err).Errorln("Error reading node constraints")
}
}
func createInfluxUserSharedService(nodeName string, service ServiceMetric) (*client.Point, error) {
tags := map[string]string{
"node": nodeName,
"service_name": service.Name,
"service_type": service.Type,
"service_image": service.Image,
}
fields := make(map[string]interface{}, len(service.Shared.UserShared))
for shared, value := range service.Shared.UserShared {
fields[shared] = value
}
if len(fields) == 0 {
return nil, errors.New("No user analytic")
}
point, err := client.NewPoint("user_shared_service", tags, fields, time.Now())
if err != nil {
return nil, err
}
log.WithField("series", "user_shared_service").Debugln("Created influx metrics")
return point, nil
}
func stopInstance(name string, instance string) {
log.Debugln("Changing instance status to stop...")
status := srv.GetServiceInstanceStatus(name, instance)
if status == enum.STOPPED {
log.Debugln("Instance already stopped")
return
}
err := srv.ChangeServiceInstanceStatus(name, instance, status, enum.STOPPED)
if err != nil {
log.WithField("err", err).Errorln("Cannot stop service instance")
return
}
log.Debugln("Updating events...")
evt_mutex.Lock()
srvEvents := events.Service[name]
srvEvents.Stop = append(srvEvents.Stop, instance)
events.Service[name] = srvEvents
evt_mutex.Unlock()
log.Debugln("Unregistering instance...")
srv.UnregisterServiceInstance(name, instance)
log.WithFields(log.Fields{
"service": name,
"id": instance,
}).Infoln("stopped instance")
}
func ReadNodeConfig(remote string, config *NodeConfig) {
configPath := remote + "/config"
err := readData(configPath, config)
if err != nil {
log.WithField("err", err).Errorln("Error reading node configuration")
}
}
func RemoveInstance(id string) {
defer mutex_instMet.Unlock()
mutex_instMet.Lock()
delete(instancesMetrics, id)
log.WithField("id", id).Debugln("Removed instance from metric collector")
}
func updateAgent(cluster string) {
configPath := c_GRU_REMOTE + cluster + "/" + c_CONFIG_REMOTE
agentConfig := cfg.Agent{}
cfg.ReadAgentConfig(configPath, &agentConfig)
cfg.SetAgent(agentConfig)
agent.UpdateStrategy()
log.WithField("agent", agentConfig).Debugln("Agent updated from remote")
}