Beispiel #1
0
func (self *EruApp) Report() {
	defer self.Client.Close()
	defer logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report stop")
	logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report start")
	for {
		select {
		case now := <-time.Tick(self.Step):
			go func() {
				info, upOk := self.updateStats()
				if isLimit {
					limitChan <- SoftLimit{upOk, self.ID, info}
				}
				if !upOk {
					logs.Info("Update mertic failed", self.Meta.ID[:12])
					return
				}
				rate := self.calcRate(info, now)
				self.saveLast(info)
				// for safe
				go self.send(rate)
			}()
		case <-self.Stop:
			return
		}
	}
}
Beispiel #2
0
func (self *EruApp) Report() {
	t := time.NewTicker(self.Step)
	defer t.Stop()
	defer self.Client.Close()
	defer logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report stop")
	logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report start")
	for {
		select {
		case now := <-t.C:
			go func() {
				if info, err := self.UpdateStats(self.ID); err == nil {
					if isLimit {
						limitChan <- SoftLimit{self.ID, info}
					}
					rate := self.CalcRate(info, now)
					self.SaveLast(info)
					go self.Send(rate)
				} else {
					logs.Info("Update mertic failed", self.ID[:12])
				}
			}()
		case <-self.Stop:
			return
		}
	}
}
Beispiel #3
0
func SetDefaultRoute(cid, gateway string) bool {
	lock.Lock()
	defer lock.Unlock()
	logs.Info("Set", cid[:12], "default route", gateway)

	container, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("RouteSetter inspect docker failed", err)
		return false
	}

	pid := strconv.Itoa(container.State.Pid)
	cmd := exec.Command("nsenter", "-t", pid, "-n", "route", "del", "default")
	if err := cmd.Run(); err != nil {
		logs.Info("Clean default route failed", err)
		return false
	}

	cmd = exec.Command("nsenter", "-t", pid, "-n", "route", "add", "default", "gw", gateway)
	if err := cmd.Run(); err != nil {
		logs.Info("RouteSetter set default route failed", err)
		return false
	}

	logs.Info("Set default route success", cid[:12], gateway)
	return true
}
Beispiel #4
0
func getContainerMeta(cid string) map[string]interface{} {
	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)

	containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
	rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn)
	if err != nil {
		logs.Info("Status get meta", err)
		return nil
	}
	var result map[string]interface{}
	if rep.IsNil() {
		return nil
	}
	if b, err := rep.Bytes(); err != nil {
		logs.Info("Status get meta", err)
		return nil
	} else {
		if err := json.Unmarshal(b, &result); err != nil {
			logs.Info("Status unmarshal meta", err)
			return nil
		}
	}
	return result
}
Beispiel #5
0
func monitor() {
	for event := range events {
		switch event.Status {
		case common.STATUS_DIE:
			logs.Debug("Status", event.Status, event.ID[:12], event.From)
			app.Remove(event.ID)
			reportContainerDeath(event.ID)
		case common.STATUS_START:
			logs.Debug("Status", event.Status, event.ID[:12], event.From)
			// if not in watching list, just ignore it
			if meta := getContainerMeta(event.ID); meta != nil && !app.Valid(event.ID) {
				container, err := g.Docker.InspectContainer(event.ID)
				if err != nil {
					logs.Info("Status inspect docker failed", err)
					break
				}
				eruApp := app.NewEruApp(container, meta)
				if eruApp == nil {
					logs.Info("Create EruApp failed")
					break
				}
				lenz.Attacher.Attach(&eruApp.Meta)
				app.Add(eruApp)
				reportContainerCure(event.ID)
			}
		}
	}
}
Beispiel #6
0
func routeWatcher() {
	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)

	subs := gore.NewSubscriptions(conn)
	defer subs.Close()
	subKey := fmt.Sprintf("eru:agent:%s:route", g.Config.HostName)
	logs.Debug("API route subscribe", subKey)
	subs.Subscribe(subKey)

	for message := range subs.Message() {
		if message == nil {
			logs.Info("API route watcher shutdown")
			break
		}
		command := string(message.Message)
		logs.Debug("API route watcher get", command)
		parser := strings.Split(command, "|")
		if len(parser) != 2 {
			logs.Info("API route watcher command invaild", command)
			continue
		}
		cid, gateway := parser[0], parser[1]
		if !network.SetDefaultRoute(cid, gateway) {
			logs.Info("Set default route failed")
		}
	}
}
Beispiel #7
0
func (self *EruApp) updateStats() bool {
	statsChan := make(chan *docker.Stats)
	opt := docker.StatsOptions{self.ID, statsChan, false, nil, time.Duration(2 * time.Second)}
	go func() {
		if err := g.Docker.Stats(opt); err != nil {
			logs.Info("Get Stats Failed", err)
		}
	}()
	stats := <-statsChan
	if stats == nil {
		return false
	}

	self.Info["cpu_user"] = stats.CPUStats.CPUUsage.UsageInUsermode
	self.Info["cpu_system"] = stats.CPUStats.CPUUsage.UsageInKernelmode
	self.Info["cpu_usage"] = stats.CPUStats.CPUUsage.TotalUsage
	//FIXME in container it will get all CPUStats
	//	for seq, d := range stats.CPUStats.CPUUsage.PercpuUsage {
	//		self.Info[fmt.Sprintf("cpu_%d", seq)] = d
	//	}
	self.Info["mem_usage"] = stats.MemoryStats.Usage
	self.Info["mem_max_usage"] = stats.MemoryStats.MaxUsage
	self.Info["mem_rss"] = stats.MemoryStats.Stats.Rss

	network, err := GetNetStats(self.Exec)
	if err != nil {
		logs.Info(err)
		return false
	}
	for k, d := range network {
		self.Info[k] = d
	}
	return true
}
Beispiel #8
0
// URL /api/container/add/
func addNewContainer(req *Request) (int, interface{}) {
	type Data struct {
		Control     string                 `json:"control"`
		ContainerID string                 `json:"container_id"`
		Meta        map[string]interface{} `json:"meta"`
	}

	data := &Data{}
	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(data)
	if err != nil {
		return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
	}

	switch data.Control {
	case "+":
		if app.Valid(data.ContainerID) {
			break
		}
		logs.Info("API status watch", data.ContainerID)
		container, err := g.Docker.InspectContainer(data.ContainerID)
		if err != nil {
			logs.Info("API status inspect docker failed", err)
			break
		}
		if eruApp := app.NewEruApp(container.ID, container.Name, data.Meta); eruApp != nil {
			app.Add(eruApp)
			lenz.Attacher.Attach(&eruApp.Meta)
		}
	}
	return http.StatusOK, JSON{"message": "ok"}
}
Beispiel #9
0
func AddRoute(cid, CIDR string, ifc string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Add route success", cid, CIDR, ifc)
	return true
}
Beispiel #10
0
func AddVLan(vethName, ips, cid string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Add VLAN device success", cid, vethName)
	return true
}
Beispiel #11
0
func SetDefaultRoute(cid, gateway string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Set default route success", cid, gateway)
	return true
}
Beispiel #12
0
func load() {
	containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		logs.Assert(err, "List containers")
	}

	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)
	containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
	logs.Debug("Status get targets from", containersKey)
	rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
	if err != nil {
		logs.Assert(err, "Status get targets")
	}

	if rep.IsNil() {
		return
	}

	targets, err := rep.Map()
	if err != nil {
		logs.Assert(err, "Status load targets")
	}

	logs.Debug("Status targets:", targets)
	logs.Info("Status load container")
	for _, container := range containers {
		if _, ok := targets[container.ID]; !ok {
			continue
		}

		status := getStatus(container.Status)
		if status != common.STATUS_START {
			reportContainerDeath(container.ID)
			continue
		}
		var meta map[string]interface{}
		if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
			logs.Info("Status load failed", err)
			continue
		}

		c, err := g.Docker.InspectContainer(container.ID)
		if err != nil {
			logs.Info("Status inspect docker failed", err)
			continue
		}

		if eruApp := app.NewEruApp(c, meta); eruApp != nil {
			lenz.Attacher.Attach(&eruApp.Meta)
			app.Add(eruApp)
			reportContainerCure(container.ID)
		}
	}
}
Beispiel #13
0
func Streamer(route *defines.Route, logstream chan *defines.Log, stdout bool) {
	var types map[string]struct{}
	var count int64 = 0
	if route.Source != nil {
		types = make(map[string]struct{})
		for _, t := range route.Source.Types {
			types[t] = struct{}{}
		}
	}
	for logline := range logstream {
		if types != nil {
			if _, ok := types[logline.Type]; !ok {
				continue
			}
		}
		logline.Tag = route.Target.AppendTag
		logline.Count = count

		switch stdout {
		case true:
			logs.Info("Debug Output", logline)
		default:
			for offset := 0; offset < route.Backends.Len(); offset++ {
				addr, err := route.Backends.Get(logline.Name, offset)
				if err != nil {
					logs.Info("Get backend failed", err, logline.Name, logline.Data)
					break
				}
				if _, ok := upstreams[addr]; !ok {
					if ups, err := NewUpStream(addr); err != nil || ups == nil {
						route.Backends.Remove(addr)
						continue
					} else {
						upstreams[addr] = ups
					}
				}
				if err := upstreams[addr].WriteData(logline); err != nil {
					upstreams[addr].Close()
					delete(upstreams, addr)
					continue
				}
				//logs.Debug("Lenz Send", logline.Name, logline.EntryPoint, logline.ID, "to", addr)
				break
			}
		}
		if count == math.MaxInt64 {
			count = 0
		} else {
			count++
		}
	}
}
Beispiel #14
0
func CloseLenz() {
	logs.Info("Close all lenz streamer")
	routes, err := Router.GetAll()
	if err != nil {
		logs.Info("Get all lenz route failed", err)
		return
	}
	for _, route := range routes {
		if !Router.Remove(route.ID) {
			logs.Info("Close lenz route failed", route.ID)
		}
	}
}
Beispiel #15
0
func softOOMKill(cid string, rate float64) {
	logs.Debug("OOM killed", cid[:12])
	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)

	key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
	if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
		logs.Info("OOM killed set flag", err)
	}
	if err := g.Docker.StopContainer(cid, 10); err != nil {
		logs.Info("OOM killed failed", cid[:12])
		return
	}
	logs.Info("OOM killed success", cid[:12])
}
Beispiel #16
0
func (self *EruApp) InitMetric() bool {
	var err error
	if self.statFile, err = os.Open(fmt.Sprintf("/proc/%d/net/dev", self.Meta.Pid)); err != nil {
		logs.Info("Open net stats failed", self.Meta.ID[:12])
		return false
	}
	info, upOk := self.updateStats()
	if !upOk {
		logs.Info("Init mertics failed", self.Meta.ID[:12])
		return false
	}
	self.Last = time.Now()
	self.saveLast(info)
	return true
}
Beispiel #17
0
func InitTransfers() {
	Transfers = consistent.New()
	for _, transfer := range Config.Metrics.Transfers {
		Transfers.Add(transfer)
	}
	logs.Info("Metrics initiated")
}
Beispiel #18
0
func HTTPServe() {
	restfulAPIServer := pat.New()

	handlers := map[string]map[string]func(*Request) (int, interface{}){
		"GET": {
			"/profile/":      profile,
			"/version/":      version,
			"/api/app/list/": listEruApps,
		},
		"POST": {
			"/api/container/add/":                    addNewContainer,
			"/api/container/:container_id/addvlan/":  addVlanForContainer,
			"/api/container/:container_id/setroute/": setRouteForContainer,
		},
	}

	for method, routes := range handlers {
		for route, handler := range routes {
			restfulAPIServer.Add(method, route, http.HandlerFunc(JSONWrapper(handler)))
		}
	}

	http.Handle("/", restfulAPIServer)
	logs.Info("API http server start at", g.Config.API.Addr)
	err := http.ListenAndServe(g.Config.API.Addr, nil)
	if err != nil {
		logs.Assert(err, "ListenAndServe: ")
	}
}
Beispiel #19
0
func (self *Metric) UpdateStats(cid string) (map[string]uint64, error) {
	info := map[string]uint64{}
	statsChan := make(chan *docker.Stats)
	doneChan := make(chan bool)
	opt := docker.StatsOptions{cid, statsChan, false, doneChan, g.timeout * time.Second}
	go func() {
		if err := g.client.Stats(opt); err != nil {
			logs.Info("Get stats failed", cid[:12], err)
		}
	}()

	var stats *docker.Stats = nil
	select {
	case stats = <-statsChan:
		if stats == nil {
			return info, errors.New("Get stats failed")
		}
	case <-time.After(g.force * time.Second):
		doneChan <- true
		return info, errors.New("Get stats timeout")
	}

	info["cpu_user"] = stats.CPUStats.CPUUsage.UsageInUsermode
	info["cpu_system"] = stats.CPUStats.CPUUsage.UsageInKernelmode
	info["cpu_usage"] = stats.CPUStats.CPUUsage.TotalUsage
	//FIXME in container it will get all CPUStats
	info["mem_usage"] = stats.MemoryStats.Usage
	info["mem_max_usage"] = stats.MemoryStats.MaxUsage
	info["mem_rss"] = stats.MemoryStats.Stats.Rss

	if err := self.getNetStats(info); err != nil {
		return info, err
	}
	return info, nil
}
Beispiel #20
0
func Limit() {
	if g.Config.Limit.Memory != 0 {
		logs.Info("App memory soft limit start")
		isLimit = true
		go calcMemoryUsage()
	}
}
Beispiel #21
0
func calcMemoryUsage() {
	for {
		select {
		case d := <-limitChan:
			if !d.flag {
				logs.Info("Get mem stats failed", d.cid)
			}
			if v, ok := d.info["mem_usage"]; ok {
				usage[d.cid] = v
			} else {
				usage[d.cid] = 0
			}
			var doCalc bool = true
			for id, _ := range Apps {
				if _, ok := usage[id]; !ok {
					doCalc = false
					break
				}
			}
			if doCalc {
				judgeMemoryUsage()
			}
		}
	}
}
Beispiel #22
0
func (self *UpStream) createUDPConn() error {
	self.scheme = "udp"
	udpAddr, err := net.ResolveUDPAddr("udp", self.addr)
	if err != nil {
		logs.Info("Resolve", self.addr, "failed", err)
		return err
	}
	conn, err := net.DialUDP("udp", nil, udpAddr)
	if err != nil {
		logs.Info("Connect backend failed", err)
		return err
	}
	self.udplog = conn
	self.Close = self.udplog.Close
	return nil
}
Beispiel #23
0
func main() {
	g.LoadConfig()
	g.InitialConn()
	g.InitTransfers()
	defer g.CloseConn()

	lenz.InitLenz()
	status.InitStatus()
	network.InitVlan()

	utils.WritePid(g.Config.PidFile)
	defer os.Remove(g.Config.PidFile)

	api.Serve()
	status.Load()
	status.StartMonitor()
	health.Check()
	app.Limit()

	var c = make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	signal.Notify(c, syscall.SIGHUP)
	signal.Notify(c, syscall.SIGKILL)
	signal.Notify(c, syscall.SIGQUIT)
	logs.Info("Eru Agent Catch", <-c)
}
Beispiel #24
0
func MakeMockedWrapper(fptr interface{}) {
	var maker = func(in []reflect.Value) []reflect.Value {
		wrapper := in[0].Elem()
		client := in[1]
		wrapperType := wrapper.Type()
		for i := 1; i < wrapperType.NumField(); i++ {
			field := wrapper.Field(i)
			fd, ok := client.Type().MethodByName(wrapperType.Field(i).Name)
			if !ok {
				logs.Info("Reflect Failed")
				continue
			}
			fdt := fd.Type
			f := reflect.MakeFunc(field.Type(), func(in []reflect.Value) []reflect.Value {
				ret := make([]reflect.Value, 0, fdt.NumOut())
				for i := 0; i < fdt.NumOut(); i++ {
					ret = append(ret, reflect.Zero(fdt.Out(i)))
				}
				return ret
			})
			field.Set(f)
		}
		return []reflect.Value{in[0]}
	}
	fn := reflect.ValueOf(fptr).Elem()
	v := reflect.MakeFunc(fn.Type(), maker)
	fn.Set(v)
}
Beispiel #25
0
func InitVlan() {
	Devices = consistent.New()
	for _, device := range g.Config.VLan.Physical {
		Devices.Add(device)
	}
	logs.Info("Vlan initiated")
}
Beispiel #26
0
func Marshal(obj interface{}) []byte {
	bytes, err := json.MarshalIndent(obj, "", "  ")
	if err != nil {
		logs.Info("Utils Marshal:", err)
	}
	return bytes
}
Beispiel #27
0
func (this *SingleConnRpcClient) insureConn() error {
	if this.rpcClient != nil {
		return nil
	}

	var err error
	var retry int = 1

	for {
		if this.rpcClient != nil {
			return nil
		}

		this.rpcClient, err = net.JsonRpcClient("tcp", this.RpcServer, this.Timeout)
		if err == nil {
			return nil
		}

		logs.Info("Metrics rpc dial fail", err)
		if retry > 5 {
			return err
		}

		time.Sleep(time.Duration(math.Pow(2.0, float64(retry))) * time.Second)
		retry++
	}
	return nil
}
Beispiel #28
0
func (this *SingleConnRpcClient) Call(method string, args interface{}, reply interface{}) error {

	this.Lock()
	defer this.Unlock()

	if err := this.insureConn(); err != nil {
		return err
	}

	timeout := time.Duration(50 * time.Second)
	done := make(chan error)

	go func() {
		err := this.rpcClient.Call(method, args, reply)
		done <- err
	}()

	select {
	case <-time.After(timeout):
		logs.Info("Metrics rpc call timeout", this.rpcClient, this.RpcServer)
		this.Close()
	case err := <-done:
		if err != nil {
			this.Close()
			return err
		}
	}

	return nil
}
Beispiel #29
0
func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp {
	name, entrypoint, ident := utils.GetAppInfo(container.Name)
	if name == "" {
		logs.Info("Container name invaild", container.Name)
		return nil
	}
	logs.Debug("Eru App", name, entrypoint, ident)

	transfer, _ := g.Transfers.Get(container.ID, 0)
	client := falcon.CreateFalconClient(
		transfer,
		time.Duration(g.Config.Metrics.Timeout)*time.Millisecond,
	)

	step := time.Duration(g.Config.Metrics.Step) * time.Second
	extend["hostname"] = g.Config.HostName
	extend["cid"] = container.ID[:12]
	extend["ident"] = ident
	tag := []string{}
	for k, v := range extend {
		tag = append(tag, fmt.Sprintf("%s=%v", k, v))
	}
	endpoint := fmt.Sprintf("%s-%s", name, entrypoint)

	meta := defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend}
	metric := metric.CreateMetric(step, client, strings.Join(tag, ","), endpoint)
	eruApp := &EruApp{meta, metric}
	return eruApp
}
Beispiel #30
0
func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp {
	name, entrypoint, ident := utils.GetAppInfo(container.Name)
	if name == "" {
		logs.Info("Container name invaild", container.Name)
		return nil
	}
	logs.Debug("Eru App", name, entrypoint, ident)

	transfer, _ := g.Transfers.Get(container.ID, 0)
	client := defines.SingleConnRpcClient{
		RpcServer: transfer,
		Timeout:   time.Duration(g.Config.Metrics.Timeout) * time.Millisecond,
	}
	step := time.Duration(g.Config.Metrics.Step) * time.Second

	extend["hostname"] = g.Config.HostName
	extend["cid"] = container.ID[:12]
	extend["ident"] = ident
	tag := []string{}
	for k, v := range extend {
		tag = append(tag, fmt.Sprintf("%s=%v", k, v))
	}
	endpoint := fmt.Sprintf("%s-%s", name, entrypoint)

	eruApp := &EruApp{
		defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend},
		defines.Metric{Step: step, Client: client, Tag: strings.Join(tag, ","), Endpoint: endpoint},
		nil,
	}

	eruApp.Stop = make(chan bool)
	return eruApp
}