Exemple #1
0
func monitor() {
	for event := range events {
		switch event.Status {
		case common.STATUS_DIE:
			logs.Debug("Status", event.Status, event.ID[:12], event.From)
			app.Remove(event.ID)
			reportContainerDeath(event.ID)
		case common.STATUS_START:
			logs.Debug("Status", event.Status, event.ID[:12], event.From)
			// if not in watching list, just ignore it
			if meta := getContainerMeta(event.ID); meta != nil && !app.Valid(event.ID) {
				container, err := g.Docker.InspectContainer(event.ID)
				if err != nil {
					logs.Info("Status inspect docker failed", err)
					break
				}
				eruApp := app.NewEruApp(container, meta)
				if eruApp == nil {
					logs.Info("Create EruApp failed")
					break
				}
				lenz.Attacher.Attach(&eruApp.Meta)
				app.Add(eruApp)
				reportContainerCure(event.ID)
			}
		}
	}
}
Exemple #2
0
func ListenAndServe() error {
	svrConfig := InitSSHConfig()
	listener, err := net.Listen("tcp", g.Config.Bind)
	if err != nil {
		logs.Assert(err, "net.Listen failed")
	}
	defer listener.Close()

	for {
		conn, err := listener.Accept()
		if err != nil {
			logs.Assert(err, "listen.Accept failed")
		}

		sshConn := &SSHConn{conn, svrConfig}

		go func() {
			if err := sshConn.serve(); err != nil {
				logs.Info("Error occured while serving", err)
				return
			}
			logs.Info("Connection closed.")
		}()
	}
}
Exemple #3
0
func setDefaultRoute(cid, gateway string, pid int) bool {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	origins, err := netns.Get()
	if err != nil {
		logs.Info("Get orignal namespace failed", err)
		return false
	}
	defer origins.Close()

	ns, err := netns.GetFromPid(pid)
	if err != nil {
		logs.Info("Get container namespace failed", err)
		return false
	}

	netns.Set(ns)
	defer ns.Close()
	defer netns.Set(origins)

	if err := delDefaultRoute(); err != nil {
		logs.Info("Delete default routing table failed", err)
		return false
	}

	if err := addDefaultRoute(gateway); err != nil {
		logs.Info("Add default route failed", err)
		return false
	}

	logs.Info("Set default route success", cid[:12], gateway)
	return true
}
Exemple #4
0
func getContainerMeta(cid string) map[string]interface{} {
	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)

	containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
	rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn)
	if err != nil {
		logs.Info("Status get meta", err)
		return nil
	}
	var result map[string]interface{}
	if rep.IsNil() {
		return nil
	}
	if b, err := rep.Bytes(); err != nil {
		logs.Info("Status get meta", err)
		return nil
	} else {
		if err := json.Unmarshal(b, &result); err != nil {
			logs.Info("Status unmarshal meta", err)
			return nil
		}
	}
	return result
}
Exemple #5
0
// URL /api/container/add/
func addNewContainer(req *Request) (int, interface{}) {
	type Data struct {
		Control     string                 `json:"control"`
		ContainerID string                 `json:"container_id"`
		Meta        map[string]interface{} `json:"meta"`
	}

	data := &Data{}
	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(data)
	if err != nil {
		return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
	}

	switch data.Control {
	case "+":
		if app.Valid(data.ContainerID) {
			break
		}
		logs.Info("API status watch", data.ContainerID)
		container, err := g.Docker.InspectContainer(data.ContainerID)
		if err != nil {
			logs.Info("API status inspect docker failed", err)
			break
		}
		if eruApp := app.NewEruApp(container, data.Meta); eruApp != nil {
			lenz.Attacher.Attach(&eruApp.Meta)
			app.Add(eruApp)
		}
	}
	return http.StatusOK, JSON{"message": "ok"}
}
Exemple #6
0
func addRoute(cid, CIDR, ifc string, pid int) bool {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	origins, err := netns.Get()
	if err != nil {
		logs.Info("Get orignal namespace failed", err)
		return false
	}
	defer origins.Close()

	ns, err := netns.GetFromPid(pid)
	if err != nil {
		logs.Info("Get container namespace failed", err)
		return false
	}

	netns.Set(ns)
	defer ns.Close()
	defer netns.Set(origins)

	if err := addRouteByLink(CIDR, ifc); err != nil {
		logs.Info("Add route failed", err)
		return false
	}

	logs.Info("Add route success", cid[:12], CIDR, ifc)
	return true
}
Exemple #7
0
func (self *EruApp) Report() {
	t := time.NewTicker(self.Step)
	defer t.Stop()
	defer self.Client.Close()
	defer logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report stop")
	logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report start")
	for {
		select {
		case now := <-t.C:
			go func() {
				if info, err := self.UpdateStats(self.ID); err == nil {
					if isLimit {
						limitChan <- SoftLimit{self.ID, info}
					}
					rate := self.CalcRate(info, now)
					self.SaveLast(info)
					go self.Send(rate)
				} else {
					logs.Info("Update mertic failed", self.ID[:12])
				}
			}()
		case <-self.Stop:
			return
		}
	}
}
Exemple #8
0
func setUpVLan(cid, ips string, pid int, veth netlink.Link) bool {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	origns, err := netns.Get()
	if err != nil {
		logs.Info("Get orignal namespace failed", err)
		return false
	}
	defer origns.Close()

	ns, err := netns.GetFromPid(pid)
	if err != nil {
		logs.Info("Get container namespace failed", err)
		return false
	}

	netns.Set(ns)
	defer ns.Close()
	defer netns.Set(origns)

	if err := BindAndSetup(veth, ips); err != nil {
		logs.Info("Bind and setup NIC failed", err)
		DelVlan(veth)
		return false
	}

	logs.Info("Add vlan device success", cid[:12])
	return true
}
Exemple #9
0
func SetDefaultRoute(cid, gateway string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Set default route success", cid, gateway)
	return true
}
Exemple #10
0
func AddRoute(cid, CIDR string, ifc string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Add route success", cid, CIDR, ifc)
	return true
}
Exemple #11
0
func AddVlan(vethName, ips, cid string) bool {
	_, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("VLanSetter inspect docker failed", err)
		return false
	}
	logs.Info("Add VLAN device success", cid, vethName)
	return true
}
Exemple #12
0
func load() {
	containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		logs.Assert(err, "List containers")
	}

	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)
	containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
	logs.Debug("Status get targets from", containersKey)
	rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
	if err != nil {
		logs.Assert(err, "Status get targets")
	}

	if rep.IsNil() {
		return
	}

	targets, err := rep.Map()
	if err != nil {
		logs.Assert(err, "Status load targets")
	}

	logs.Debug("Status targets:", targets)
	logs.Info("Status load container")
	for _, container := range containers {
		if _, ok := targets[container.ID]; !ok {
			continue
		}

		status := getStatus(container.Status)
		if status != common.STATUS_START {
			reportContainerDeath(container.ID)
			continue
		}
		var meta map[string]interface{}
		if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
			logs.Info("Status load failed", err)
			continue
		}

		c, err := g.Docker.InspectContainer(container.ID)
		if err != nil {
			logs.Info("Status inspect docker failed", err)
			continue
		}

		if eruApp := app.NewEruApp(c, meta); eruApp != nil {
			lenz.Attacher.Attach(&eruApp.Meta)
			app.Add(eruApp)
			reportContainerCure(container.ID)
		}
	}
}
Exemple #13
0
func CloseLenz() {
	logs.Info("Close all lenz streamer")
	routes, err := Router.GetAll()
	if err != nil {
		logs.Info("Get all lenz route failed", err)
		return
	}
	for _, route := range routes {
		if !Router.Remove(route.ID) {
			logs.Info("Close lenz route failed", route.ID)
		}
	}
}
Exemple #14
0
// URL /api/container/:container_id/addcalico/
func addCalicoForContainer(req *Request) (int, interface{}) {
	type Endpoint struct {
		Nid     int    `json:"nid"`
		Profile string `json:"profile"`
		IP      string `json:"ip"`
		Append  bool   `json:"append"`
	}
	type Result struct {
		Succ        int    `json:"succ"`
		ContainerID string `json:"container_id"`
		IP          string `json:"ip"`
		Err         string `json:"err"`
	}

	if g.Config.VLan.Calico == "" {
		return http.StatusBadRequest, JSON{"message": "Agent not enable calico support"}
	}

	cid := req.URL.Query().Get(":container_id")
	env := os.Environ()
	env = append(env, fmt.Sprintf("ETCD_AUTHORITY=%s", g.Config.VLan.Calico))

	endpoints := []Endpoint{}
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(&endpoints); err != nil {
		return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
	}

	rv := []Result{}
	for seq, endpoint := range endpoints {
		vethName := fmt.Sprintf("%s%d.%d", common.VLAN_PREFIX, endpoint.Nid, seq)
		if err := network.AddCalico(env, endpoint.Append, cid, vethName, endpoint.IP); err != nil {
			rv = append(rv, Result{Succ: 0, ContainerID: cid, IP: endpoint.IP, Err: err.Error()})
			logs.Info("API calico add interface failed", err)
			continue
		}

		//TODO remove when eru-core support ACL
		// currently only one profile is used
		if err := network.BindCalicoProfile(env, cid, endpoint.Profile); err != nil {
			rv = append(rv, Result{Succ: 0, ContainerID: cid, IP: endpoint.IP, Err: err.Error()})
			logs.Info("API calico add profile failed", err)
			continue
		}

		rv = append(rv, Result{Succ: 1, ContainerID: cid, IP: endpoint.IP})
	}
	return http.StatusOK, rv
}
Exemple #15
0
func softOOMKill(cid string, rate float64) {
	logs.Debug("OOM killed", cid[:12])
	conn := g.GetRedisConn()
	defer g.ReleaseRedisConn(conn)

	key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
	if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
		logs.Info("OOM killed set flag", err)
	}
	if err := g.Docker.StopContainer(cid, 10); err != nil {
		logs.Info("OOM killed failed", cid[:12])
		return
	}
	logs.Info("OOM killed success", cid[:12])
}
Exemple #16
0
func Limit() {
	if g.Config.Limit.Memory != 0 {
		logs.Info("App memory soft limit start")
		isLimit = true
		go calcMemoryUsage()
	}
}
Exemple #17
0
func HTTPServe() {
	restfulAPIServer := pat.New()

	handlers := map[string]map[string]func(*Request) (int, interface{}){
		"GET": {
			"/profile/":      profile,
			"/version/":      version,
			"/api/app/list/": listEruApps,
		},
		"POST": {
			"/api/container/add/":                     addNewContainer,
			"/api/container/:container_id/addvlan/":   addVlanForContainer,
			"/api/container/:container_id/addcalico/": addCalicoForContainer,
			"/api/container/:container_id/setroute/":  setRouteForContainer,
			"/api/container/:container_id/addroute/":  addRouteForContainer,
			"/api/eip/bind/":                          bindEIP,
			"/api/eip/release/":                       releaseEIP,
			"/api/container/publish/":                 publishContainer,
			"/api/container/unpublish/":               unpublishContainer,
		},
	}

	for method, routes := range handlers {
		for route, handler := range routes {
			restfulAPIServer.Add(method, route, http.HandlerFunc(JSONWrapper(handler)))
		}
	}

	http.Handle("/", restfulAPIServer)
	logs.Info("API http server start at", g.Config.API.Addr)
	err := http.ListenAndServe(g.Config.API.Addr, nil)
	if err != nil {
		logs.Assert(err, "ListenAndServe: ")
	}
}
Exemple #18
0
func SetDefaultRoute(cid, gateway string) bool {
	lock.Lock()
	defer lock.Unlock()

	logs.Info("Set", cid[:12], "default route", gateway)

	container, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("RouteSetter inspect docker failed", err)
		return false
	}

	pid := container.State.Pid

	return setDefaultRoute(cid, gateway, pid)
}
Exemple #19
0
func AddRoute(cid, CIDR, ifc string) bool {
	lock.Lock()
	defer lock.Unlock()

	logs.Info("Add", cid[:12], "route", CIDR, ifc)

	container, err := g.Docker.InspectContainer(cid)
	if err != nil {
		logs.Info("RouteSetter inspect docker failed", err)
		return false
	}

	pid := container.State.Pid

	return addRoute(cid, CIDR, ifc, pid)
}
Exemple #20
0
// URL /api/eip/release/
func releaseEIP(req *Request) (int, interface{}) {
	type EIP struct {
		ID int `json:"id"`
	}
	type Result struct {
		Succ int    `json:"succ"`
		Err  string `json:"err"`
		Veth string `json:"id"`
	}

	eips := []EIP{}
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(&eips); err != nil {
		return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
	}

	rv := []Result{}
	for _, eip := range eips {
		vethName := fmt.Sprintf("%s%d", common.VLAN_PREFIX, eip.ID)
		if err := network.DelMacVlanDevice(vethName); err != nil {
			rv = append(rv, Result{Succ: 0, Veth: vethName, Err: err.Error()})
			logs.Info("Release EIP failed", err, vethName)
			continue
		}
		rv = append(rv, Result{Succ: 1, Veth: vethName})
	}

	return http.StatusOK, rv
}
Exemple #21
0
func Marshal(obj interface{}) []byte {
	bytes, err := json.MarshalIndent(obj, "", "  ")
	if err != nil {
		logs.Info("Utils Marshal:", err)
	}
	return bytes
}
Exemple #22
0
func main() {
	g.LoadConfig()
	g.InitialConn()
	g.InitTransfers()
	defer g.CloseConn()

	lenz.InitLenz()
	status.InitStatus()
	network.InitVlan()
	defer lenz.CloseLenz()

	utils.WritePid(g.Config.PidFile)
	defer os.Remove(g.Config.PidFile)

	app.Limit()
	app.Metric()
	api.Serve()
	status.Start()
	health.Check()

	var c = make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	signal.Notify(c, syscall.SIGHUP)
	signal.Notify(c, syscall.SIGKILL)
	signal.Notify(c, syscall.SIGQUIT)
	logs.Info("Eru Agent Catch", <-c)
}
Exemple #23
0
func InitTransfers() {
	Transfers = consistent.New()
	for _, transfer := range Config.Metrics.Transfers {
		Transfers.Add(transfer)
	}
	logs.Info("Transfers initiated")
}
Exemple #24
0
func InitVlan() {
	Devices = consistent.New()
	for _, device := range g.Config.VLan.Physical {
		Devices.Add(device)
	}
	logs.Info("Vlan initiated")
}
Exemple #25
0
func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp {
	name, entrypoint, ident := utils.GetAppInfo(container.Name)
	if name == "" {
		logs.Info("Container name invaild", container.Name)
		return nil
	}
	logs.Debug("Eru App", name, entrypoint, ident)

	transfer, _ := g.Transfers.Get(container.ID, 0)
	client := falcon.CreateFalconClient(
		transfer,
		time.Duration(g.Config.Metrics.Timeout)*time.Millisecond,
	)

	step := time.Duration(g.Config.Metrics.Step) * time.Second
	extend["hostname"] = g.Config.HostName
	extend["cid"] = container.ID[:12]
	extend["ident"] = ident
	tag := []string{}
	for k, v := range extend {
		tag = append(tag, fmt.Sprintf("%s=%v", k, v))
	}
	endpoint := fmt.Sprintf("%s-%s", name, entrypoint)

	meta := defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend}
	metric := metric.CreateMetric(step, client, strings.Join(tag, ","), endpoint)
	eruApp := &EruApp{meta, metric}
	return eruApp
}
Exemple #26
0
// URL /api/eip/bind/
func bindEIP(req *Request) (int, interface{}) {
	type EIP struct {
		ID        int    `json:"id"`
		IP        string `json:"ip"`
		Broadcast string `json:"broadcast"`
	}
	type Result struct {
		Succ int    `json:"succ"`
		Err  string `json:"err"`
		IP   string `json:"ip"`
	}

	eips := []EIP{}
	decoder := json.NewDecoder(req.Body)
	if err := decoder.Decode(&eips); err != nil {
		return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
	}

	rv := []Result{}
	for _, eip := range eips {
		vethName := fmt.Sprintf("%s%d", common.VLAN_PREFIX, eip.ID)
		veth, err := network.AddMacVlanDevice(vethName, vethName)
		if err != nil {
			rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()})
			logs.Info("API add EIP failed", err)
			continue
		}

		if err := network.BindAndSetup(veth, eip.IP); err != nil {
			rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()})
			network.DelVlan(veth)
			logs.Info("API bind EIP failed", err)
			continue
		}

		if err := network.SetBroadcast(vethName, eip.Broadcast); err != nil {
			rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()})
			network.DelVlan(veth)
			logs.Info("API set broadcast failed", err)
			continue
		}

		rv = append(rv, Result{Succ: 1, IP: eip.IP})
	}

	return http.StatusOK, rv
}
Exemple #27
0
func Metric() {
	metric.SetGlobalSetting(
		g.Docker, time.Duration(common.STATS_TIMEOUT),
		time.Duration(common.STATS_FORCE_DONE),
		common.VLAN_PREFIX, common.DEFAULT_BR,
	)
	logs.Info("Metrics initiated")
}
Exemple #28
0
func (self *UpStream) createUDPConn() error {
	self.scheme = "udp"
	udpAddr, err := net.ResolveUDPAddr("udp", self.addr)
	if err != nil {
		logs.Info("Resolve", self.addr, "failed", err)
		return err
	}
	conn, err := net.DialUDP("udp", nil, udpAddr)
	if err != nil {
		logs.Info("Connect backend failed", err)
		return err
	}
	self.conn = conn
	self.encoder = json.NewEncoder(conn)
	self.Close = conn.Close
	return nil
}
Exemple #29
0
func DelMacVlanDevice(vethName string) error {
	logs.Info("Release macvlan device", vethName)
	link, err := netlink.LinkByName(vethName)
	if err != nil {
		return err
	}
	DelVlan(link)
	return nil
}
Exemple #30
0
func LoadConfig() {
	var configPath string
	var version bool
	flag.BoolVar(&logs.Mode, "DEBUG", false, "enable debug")
	flag.StringVar(&configPath, "c", "agent.yaml", "config file")
	flag.BoolVar(&version, "v", false, "show version")
	flag.Parse()
	if version {
		logs.Info("Version", common.VERSION)
		os.Exit(0)
	}
	load(configPath)
}