func stopInstance(name string, instance string) { log.Debugln("Changing instance status to stop...") status := srv.GetServiceInstanceStatus(name, instance) if status == enum.STOPPED { log.Debugln("Instance already stopped") return } err := srv.ChangeServiceInstanceStatus(name, instance, status, enum.STOPPED) if err != nil { log.WithField("err", err).Errorln("Cannot stop service instance") return } log.Debugln("Updating events...") evt_mutex.Lock() srvEvents := events.Service[name] srvEvents.Stop = append(srvEvents.Stop, instance) events.Service[name] = srvEvents evt_mutex.Unlock() log.Debugln("Unregistering instance...") srv.UnregisterServiceInstance(name, instance) log.WithFields(log.Fields{ "service": name, "id": instance, }).Infoln("stopped instance") }
func notifyRemoval() { log.Debugln("Checking removal notification...") if chn.NeedsRemovalNotification() { log.Debugln("Needs removal notification. Sending message...") chn.GetRemovalChannel() <- struct{}{} log.Debugln("Removal notified") chn.SetRemovalNotification(false) } }
func GetCoresAvailable(number int) (string, bool) { defer runtime.Gosched() cores_str := make([]string, 0, number) mutex_cpu.RLock() for i := 0; i < len(resources.CPU.Cores); i++ { if resources.CPU.Cores[i] == true { cores_str = append(cores_str, strconv.Itoa(i)) } if len(cores_str) >= number { break } } if len(cores_str) < number { log.Debugln("Error getting available cores: number of free cores < ", number) mutex_cpu.RUnlock() return "", false } mutex_cpu.RUnlock() cores := strings.Join(cores_str, ",") log.WithFields(log.Fields{ "number": number, "cores": cores, }).Debugln("Getting available cores") return cores, true }
func keepAlive(name string, id string) { var err error discoveryConf := cfg.GetAgentDiscovery() ticker := time.NewTicker(time.Duration(discoveryConf.TTL-1) * time.Second) opt := discovery.Options{ "TTL": time.Duration(discoveryConf.TTL) * time.Second, } ch_stop := ch.CreateInstanceChannel(id) isntanceKey := discoveryConf.AppRoot + "/" + name + "/" + id instanceValue := addressMap[id] for { select { case <-ticker.C: err = discovery.Set(isntanceKey, instanceValue, opt) if err != nil { log.WithFields(log.Fields{ "service": name, "instance": id, "address": instanceValue, "err": err, }).Errorln("Error keeping instance alive") } case <-ch_stop: log.Debugln("Stopping keep alive routine") return } } }
func (db *influxdb) Initialize(config map[string]interface{}) error { var err error db.config = &influxdbConfig{} err = utils.FillStruct(db.config, config) if err != nil { return err } log.Debugln("Initializing influxdb at address: ", db.config.Url) db.influx, err = client.NewHTTPClient(client.HTTPConfig{ Addr: db.config.Url, Username: db.config.Username, Password: db.config.Password, }) if err != nil { return err } db.batch, err = client.NewBatchPoints(client.BatchPointsConfig{ Database: db.config.DbName, Precision: "s", }) if err != nil { return err } return nil }
func updateFriendsData(nFriends int) error { var err error log.Debugln("Updating friends data") err = clearFriendsData() if err != nil { return err } peers := getAllPeers() log.WithField("peers", len(peers)).Debugln("Number of peers") if len(peers) == 0 { return ErrNoPeers } friends, err := chooseRandomFriends(peers, nFriends) if err != nil { return err } log.WithField("friends", friends).Debugln("Friends to connect with") friendsData, err := getFriendsData(friends) if err != nil { return err } clusterData, err := mergeSharedData(friendsData) if err != nil { return err } data.SaveSharedCluster(clusterData) return nil }
func collectMetrics() { log.Debugln("Collecting metrics") updateMetrics() err := storeMetrics(metrics) if err != nil { log.WithField("errr", err).Errorln("Error collecting agent metrics") } }
func createMemory(memory string) int64 { memInBytes, err := utils.RAMInBytes(memory) if err != nil { log.Debugln("Creating Host config: Memory limit not specified") return 0 } return memInBytes }
func listen() { ch_action := ch.GetActionChannel() for { select { case msg := <-ch_action: log.Debugln("Received action message") executeActions(msg.Target, msg.Actions) } } }
func nameExist(nodeName string, clusterName string) bool { names := cluster.ListNodes(clusterName, false) log.Debugln("Nodes list: ", names) for name, _ := range names { if name == nodeName { return true } } return false }
func GetNodes(clusterName string, onlyActive bool) []cfg.Node { remote := c_GRU_REMOTE + clusterName + "/nodes" nodes := cfg.ReadNodes(remote) if onlyActive { active := []cfg.Node{} for _, node := range nodes { if node.Active { active = append(active, node) } } log.Debugln("Active nodes: ", active) return active } log.Debugln("Nodes: ", nodes) return nodes }
func initializeNode(nodeName string, clusterName string) { if nodeName == "random_name" { nodeName = utils.GetRandomName(0) } counter := -2 for nameExist(nodeName, clusterName) { nodeName = utils.GetRandomName(counter) counter++ } log.Debugln("Node name: ", nodeName) node.CreateNode(nodeName, res.GetResources()) }
func createCpusetCpus(cpusetcpus string, cores int) string { if cpusetcpus == "" { if cores < 1 { log.Warnln("Number of requested CPUs = 0. Setting to 1") cores = 1 } if assigned, ok := res.GetCoresAvailable(cores); ok { cpusetcpus = assigned } else { log.Debugln("Error setting cpusetcpus in hostconfig") } } return cpusetcpus }
func updateCommand(cmd Command) { log.Debugln("Updating ", cmd.Target) switch cmd.Target { case "all": cluster := cmd.Object.(string) updateAll(cluster) case "agent": cluster := cmd.Object.(string) updateAgent(cluster) case "services": cluster := cmd.Object.(string) updateServices(cluster) case "policy": cluster := cmd.Object.(string) updatePolicy(cluster) case "node-base-services": data := cmd.Object.([]interface{}) upd := []string{} for _, item := range data { upd = append(upd, item.(string)) } constraints := cfg.GetNodeConstraints() constraints.BaseServices = upd cfg.WriteNodeConstraints(cfg.GetNodeConfig().Remote, *constraints) case "node-cpumin": upd := cmd.Object.(float64) constraints := cfg.GetNodeConstraints() constraints.CpuMin = upd cfg.WriteNodeConstraints(cfg.GetNodeConfig().Remote, *constraints) case "node-cpumax": upd := cmd.Object.(float64) constraints := cfg.GetNodeConstraints() constraints.CpuMax = upd cfg.WriteNodeConstraints(cfg.GetNodeConfig().Remote, *constraints) case "service-constraints": name := cmd.Object.(string) srv, _ := service.GetServiceByName(name) upd := cfg.ReadService(srv.Remote) srv.Constraints = upd.Constraints default: log.WithField("target", cmd.Target).Errorln("Unrecognized target for command update") } }
func RunLoop(loopTimeInterval int) { // Start the metric collector metric.StartMetricCollector() // Set the ticker for the periodic execution ticker := time.NewTicker(time.Duration(loopTimeInterval) * time.Second) log.Infoln("Running autonomic loop") for { select { case <-ticker.C: stats := monitor.Run() analytics := analyzer.Run(stats) policy := planner.Run(analytics) executor.Run(policy) log.Infoln("-------------------------") case <-ch_err: log.Debugln("Error running autonomic loop") } } }
func collector(contLog io.ReadCloser, ch_entry chan logEntry) { var err error var line []byte var data logEntry scanner := bufio.NewScanner(contLog) for scanner.Scan() { line = scanner.Bytes() if regex.Match(line) { data, err = getDataFromLogLine(string(line)) if err != nil { log.WithField("err", err).Errorln("Error parsing container logs") } else { ch_entry <- data } } } if err = scanner.Err(); err != nil { log.WithField("err", err).Errorln("Error in scanner.") } log.Debugln("Stopped collector") }
func RegisterCluster(name string, id string) { var err error err = discovery.Register(c_GRU_REMOTE+name+"/uuid", id) if err != nil { log.Errorln("Error registering cluster") } log.Debugln("Created cluster forder: ", name) opt := discovery.Options{"Dir": true} err = discovery.Set(c_GRU_REMOTE+name+"/"+c_NODES_REMOTE, "", opt) if err != nil { log.Errorln("Error creating nodes folder") } log.Debugln("Created nodes folder") err = discovery.Set(c_GRU_REMOTE+name+"/"+c_SERVICES_REMOTE, "", opt) if err != nil { log.Errorln("Error creating services folder") } log.Debugln("Created services folder") err = discovery.Set(c_GRU_REMOTE+name+"/"+c_CONFIG_REMOTE, "empty", discovery.Options{}) if err != nil { log.Errorln("Error creating config key") } log.Debugln("Created config key") err = discovery.Set(c_GRU_REMOTE+name+"/"+c_POLICY_REMOTE, "empty", discovery.Options{}) if err != nil { log.Errorln("Error creating policy key") } log.Debugln("Created policy key") err = discovery.Set(c_GRU_REMOTE+name+"/"+c_ANALYTIC_REMOTE, "", opt) if err != nil { log.Errorln("Error creating analytics folder") } log.Debugln("Created analytics folder") }
func StartCollector(contLog io.ReadCloser) { log.Debugln("starting collector") go collector(contLog, ch_entry) }
func updateMetrics() { var err error metrics = newMetrics() metrics.Node.UUID = cfg.GetNodeConfig().UUID metrics.Node.Name = cfg.GetNodeConfig().Name metrics.Node.Resources.CPU.Total = res.GetResources().CPU.Total metrics.Node.Resources.CPU.Availabe = res.GetResources().CPU.Total - res.GetResources().CPU.Used localShared, err := data.GetSharedLocal() if err != nil { log.WithField("err", err).Warnln("Cannot update node active metric") } else { metrics.Node.ActiveServices = len(localShared.System.ActiveServices) } for _, name := range service.List() { srv, _ := service.GetServiceByName(name) srv_metrics := metrics.Service[name] srv_metrics.Name = name srv_metrics.Image = srv.Image srv_metrics.Type = srv.Type srv_metrics.Instances.All = len(srv.Instances.All) srv_metrics.Instances.Pending = len(srv.Instances.Pending) srv_metrics.Instances.Running = len(srv.Instances.Running) srv_metrics.Instances.Paused = len(srv.Instances.Paused) srv_metrics.Instances.Stopped = len(srv.Instances.Stopped) stats, err := data.GetStats() if err != nil { log.WithFields(log.Fields{ "err": err, "service": name, }).Warnln("Cannot update stats metrics") } else { if srv_stats, ok := stats.Metrics.Service[name]; ok { srv_metrics.Stats = srv_stats } else { log.Warnln("Cannot find stats metrics for service ", name) } metrics.Node.Stats = stats.Metrics.System } analytics, err := data.GetAnalytics() if err != nil { log.WithFields(log.Fields{ "err": err, "service": name, }).Warnln("Cannot update analytics metrics") } else { if srv_analytisc, ok := analytics.Service[name]; ok { srv_metrics.Analytics = srv_analytisc } else { log.Debugln("Cannot find analytics metrics for service ", name) } } shared, err := data.GetSharedCluster() if err != nil { log.WithFields(log.Fields{ "err": err, "service": name, }).Warnln("Cannot update shared data metrics") } else { if srv_shared, ok := shared.Service[name]; ok { srv_metrics.Shared = srv_shared.Data } } metrics.Service[name] = srv_metrics } plc, err := data.GetPolicy() if err != nil { log.WithField("err", err).Warnln("Cannot update plans metrics") } else { metrics.Policy.Name = plc.Name metrics.Policy.Weight = plc.Weight } }
func waitForRemoval() { log.Debugln("Waiting for removal confirmation...") <-ch.GetRemovalChannel() log.Debugln("Remove complete") }
// Events are: attach, commit, copy, create, destroy, die, exec_create, exec_start, export, kill, oom, pause, rename, resize, restart, start, stop, top, unpause, update func eventCallback(event *dockerclient.Event, ec chan error, args ...interface{}) { log.Debugln("Received event") // By now we do not handle events with type != container if event.Type != "container" { log.WithField("type", event.Type).Debugln("Received event with type different from 'container'") return } service, err := srv.GetServiceByImage(event.From) if err != nil { log.WithFields(log.Fields{ "err": err, "event": event, }).Warnln("Cannot handle event") return } e := evt.Event{ Service: service.Name, Image: event.From, Instance: event.ID, Type: event.Type, } switch event.Status { case "create": log.WithField("image", e.Image).Debugln("Received create signal") evt.HandleCreateEvent(e) container.Docker().Client.StartMonitorStats(e.Instance, statCallBack, ch_mnt_stats_err) case "start": log.WithField("image", e.Image).Debugln("Received start signal") if _, ok := instBuffer[e.Instance]; !ok { instBuffer[e.Instance] = instanceMetricBuffer{ cpuInst: utils.BuildBuffer(c_B_SIZE), cpuSys: utils.BuildBuffer(c_B_SIZE), } } e.Status = enum.PENDING evt.HanldeStartEvent(e) mtr.AddInstance(e.Instance) if enableLogReading { startMonitorLog(event.ID) } case "stop": log.WithField("image", e.Image).Debugln("Received stop signal") case "kill": log.WithField("image", e.Image).Debugln("Received kill signal") case "die": log.WithField("image", e.Image).Debugln("Received die signal") delete(instBuffer, e.Instance) mtr.RemoveInstance(e.Instance) evt.HandleStopEvent(e) case "destroy": log.WithField("id", e.Instance).Debugln("Received destroy signal") evt.HandleRemoveEvent(e) default: log.WithFields(log.Fields{ "err": "event not handled", "event": event.Status, "image": event.From, }).Debugln("Received unknown signal") } log.Debugln("Event handled") }
func SetServiceInstanceResources(name string, id string) { var err error log.Debugln("Setting new instance resources") // This is needed otherwise dockerclient does not // return the correct container information time.Sleep(100 * time.Millisecond) info, err := container.Docker().Client.InspectContainer(id) if err != nil { log.WithFields(log.Fields{ "id": id, "err": err, }).Errorln("Error setting instance resources") } cpusetcpus := info.HostConfig.CpusetCpus portBindings, err := container.GetPortBindings(id) if err != nil { log.WithFields(log.Fields{ "service": name, "id": id, "err": err, }).Errorln("Error getting instance port binding") } log.WithFields(log.Fields{ "service": name, "cpusetcpus": cpusetcpus, "portbindings": portBindings, }).Debugln("New instance respources") err = CheckAndSetSpecificCores(cpusetcpus, id) if err != nil { log.WithFields(log.Fields{ "service": name, "id": id, "cpus": cpusetcpus, "err": err, }).Errorln("Error assigning CPU resources to new instance") } err = AssignSpecifiPortsToService(name, id, portBindings) if err != nil { log.WithFields(log.Fields{ "service": name, "id": id, "bindings": portBindings, "err": err, }).Errorln("Error assigning port resources to new instance") } guestPort := service.GetDiscoveryPort(name) if hostPorts, ok := portBindings[guestPort]; ok { if len(hostPorts) > 0 { service.SaveInstanceAddress(id, hostPorts[0]) log.WithFields(log.Fields{ "service": name, "instance": id, "guest": guestPort, "host": hostPorts[0], }).Debugln("Saved instance address") } else { log.WithFields(log.Fields{ "service": name, "instance": id, "guest": guestPort, }).Debugln("Cannot register instance address: host ports < 0") } } else { log.WithFields(log.Fields{ "service": name, "instance": id, "guest": guestPort, }).Debugln("Cannot register instance address: no bindings") } }
func deactivateNode() { log.Debugln("Deactivating node") cfg.ToggleActiveNode() cfg.WriteNodeActive(cfg.GetNodeConfig().Remote, false) }
func activateNode() { log.Debugln("Activating node") cfg.ToggleActiveNode() cfg.WriteNodeActive(cfg.GetNodeConfig().Remote, true) }