func monitor() { for event := range events { switch event.Status { case common.STATUS_DIE: logs.Debug("Status", event.Status, event.ID[:12], event.From) app.Remove(event.ID) reportContainerDeath(event.ID) case common.STATUS_START: logs.Debug("Status", event.Status, event.ID[:12], event.From) // if not in watching list, just ignore it if meta := getContainerMeta(event.ID); meta != nil && !app.Valid(event.ID) { container, err := g.Docker.InspectContainer(event.ID) if err != nil { logs.Info("Status inspect docker failed", err) break } eruApp := app.NewEruApp(container, meta) if eruApp == nil { logs.Info("Create EruApp failed") break } lenz.Attacher.Attach(&eruApp.Meta) app.Add(eruApp) reportContainerCure(event.ID) } } } }
func ListenAndServe() error { svrConfig := InitSSHConfig() listener, err := net.Listen("tcp", g.Config.Bind) if err != nil { logs.Assert(err, "net.Listen failed") } defer listener.Close() for { conn, err := listener.Accept() if err != nil { logs.Assert(err, "listen.Accept failed") } sshConn := &SSHConn{conn, svrConfig} go func() { if err := sshConn.serve(); err != nil { logs.Info("Error occured while serving", err) return } logs.Info("Connection closed.") }() } }
func setDefaultRoute(cid, gateway string, pid int) bool { runtime.LockOSThread() defer runtime.UnlockOSThread() origins, err := netns.Get() if err != nil { logs.Info("Get orignal namespace failed", err) return false } defer origins.Close() ns, err := netns.GetFromPid(pid) if err != nil { logs.Info("Get container namespace failed", err) return false } netns.Set(ns) defer ns.Close() defer netns.Set(origins) if err := delDefaultRoute(); err != nil { logs.Info("Delete default routing table failed", err) return false } if err := addDefaultRoute(gateway); err != nil { logs.Info("Add default route failed", err) return false } logs.Info("Set default route success", cid[:12], gateway) return true }
func getContainerMeta(cid string) map[string]interface{} { conn := g.GetRedisConn() defer g.ReleaseRedisConn(conn) containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName) rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn) if err != nil { logs.Info("Status get meta", err) return nil } var result map[string]interface{} if rep.IsNil() { return nil } if b, err := rep.Bytes(); err != nil { logs.Info("Status get meta", err) return nil } else { if err := json.Unmarshal(b, &result); err != nil { logs.Info("Status unmarshal meta", err) return nil } } return result }
// URL /api/container/add/ func addNewContainer(req *Request) (int, interface{}) { type Data struct { Control string `json:"control"` ContainerID string `json:"container_id"` Meta map[string]interface{} `json:"meta"` } data := &Data{} decoder := json.NewDecoder(req.Body) err := decoder.Decode(data) if err != nil { return http.StatusBadRequest, JSON{"message": "wrong JSON format"} } switch data.Control { case "+": if app.Valid(data.ContainerID) { break } logs.Info("API status watch", data.ContainerID) container, err := g.Docker.InspectContainer(data.ContainerID) if err != nil { logs.Info("API status inspect docker failed", err) break } if eruApp := app.NewEruApp(container, data.Meta); eruApp != nil { lenz.Attacher.Attach(&eruApp.Meta) app.Add(eruApp) } } return http.StatusOK, JSON{"message": "ok"} }
func addRoute(cid, CIDR, ifc string, pid int) bool { runtime.LockOSThread() defer runtime.UnlockOSThread() origins, err := netns.Get() if err != nil { logs.Info("Get orignal namespace failed", err) return false } defer origins.Close() ns, err := netns.GetFromPid(pid) if err != nil { logs.Info("Get container namespace failed", err) return false } netns.Set(ns) defer ns.Close() defer netns.Set(origins) if err := addRouteByLink(CIDR, ifc); err != nil { logs.Info("Add route failed", err) return false } logs.Info("Add route success", cid[:12], CIDR, ifc) return true }
func (self *EruApp) Report() { t := time.NewTicker(self.Step) defer t.Stop() defer self.Client.Close() defer logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report stop") logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report start") for { select { case now := <-t.C: go func() { if info, err := self.UpdateStats(self.ID); err == nil { if isLimit { limitChan <- SoftLimit{self.ID, info} } rate := self.CalcRate(info, now) self.SaveLast(info) go self.Send(rate) } else { logs.Info("Update mertic failed", self.ID[:12]) } }() case <-self.Stop: return } } }
func setUpVLan(cid, ips string, pid int, veth netlink.Link) bool { runtime.LockOSThread() defer runtime.UnlockOSThread() origns, err := netns.Get() if err != nil { logs.Info("Get orignal namespace failed", err) return false } defer origns.Close() ns, err := netns.GetFromPid(pid) if err != nil { logs.Info("Get container namespace failed", err) return false } netns.Set(ns) defer ns.Close() defer netns.Set(origns) if err := BindAndSetup(veth, ips); err != nil { logs.Info("Bind and setup NIC failed", err) DelVlan(veth) return false } logs.Info("Add vlan device success", cid[:12]) return true }
func SetDefaultRoute(cid, gateway string) bool { _, err := g.Docker.InspectContainer(cid) if err != nil { logs.Info("VLanSetter inspect docker failed", err) return false } logs.Info("Set default route success", cid, gateway) return true }
func AddRoute(cid, CIDR string, ifc string) bool { _, err := g.Docker.InspectContainer(cid) if err != nil { logs.Info("VLanSetter inspect docker failed", err) return false } logs.Info("Add route success", cid, CIDR, ifc) return true }
func AddVlan(vethName, ips, cid string) bool { _, err := g.Docker.InspectContainer(cid) if err != nil { logs.Info("VLanSetter inspect docker failed", err) return false } logs.Info("Add VLAN device success", cid, vethName) return true }
func load() { containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true}) if err != nil { logs.Assert(err, "List containers") } conn := g.GetRedisConn() defer g.ReleaseRedisConn(conn) containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName) logs.Debug("Status get targets from", containersKey) rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn) if err != nil { logs.Assert(err, "Status get targets") } if rep.IsNil() { return } targets, err := rep.Map() if err != nil { logs.Assert(err, "Status load targets") } logs.Debug("Status targets:", targets) logs.Info("Status load container") for _, container := range containers { if _, ok := targets[container.ID]; !ok { continue } status := getStatus(container.Status) if status != common.STATUS_START { reportContainerDeath(container.ID) continue } var meta map[string]interface{} if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil { logs.Info("Status load failed", err) continue } c, err := g.Docker.InspectContainer(container.ID) if err != nil { logs.Info("Status inspect docker failed", err) continue } if eruApp := app.NewEruApp(c, meta); eruApp != nil { lenz.Attacher.Attach(&eruApp.Meta) app.Add(eruApp) reportContainerCure(container.ID) } } }
func CloseLenz() { logs.Info("Close all lenz streamer") routes, err := Router.GetAll() if err != nil { logs.Info("Get all lenz route failed", err) return } for _, route := range routes { if !Router.Remove(route.ID) { logs.Info("Close lenz route failed", route.ID) } } }
// URL /api/container/:container_id/addcalico/ func addCalicoForContainer(req *Request) (int, interface{}) { type Endpoint struct { Nid int `json:"nid"` Profile string `json:"profile"` IP string `json:"ip"` Append bool `json:"append"` } type Result struct { Succ int `json:"succ"` ContainerID string `json:"container_id"` IP string `json:"ip"` Err string `json:"err"` } if g.Config.VLan.Calico == "" { return http.StatusBadRequest, JSON{"message": "Agent not enable calico support"} } cid := req.URL.Query().Get(":container_id") env := os.Environ() env = append(env, fmt.Sprintf("ETCD_AUTHORITY=%s", g.Config.VLan.Calico)) endpoints := []Endpoint{} decoder := json.NewDecoder(req.Body) if err := decoder.Decode(&endpoints); err != nil { return http.StatusBadRequest, JSON{"message": "wrong JSON format"} } rv := []Result{} for seq, endpoint := range endpoints { vethName := fmt.Sprintf("%s%d.%d", common.VLAN_PREFIX, endpoint.Nid, seq) if err := network.AddCalico(env, endpoint.Append, cid, vethName, endpoint.IP); err != nil { rv = append(rv, Result{Succ: 0, ContainerID: cid, IP: endpoint.IP, Err: err.Error()}) logs.Info("API calico add interface failed", err) continue } //TODO remove when eru-core support ACL // currently only one profile is used if err := network.BindCalicoProfile(env, cid, endpoint.Profile); err != nil { rv = append(rv, Result{Succ: 0, ContainerID: cid, IP: endpoint.IP, Err: err.Error()}) logs.Info("API calico add profile failed", err) continue } rv = append(rv, Result{Succ: 1, ContainerID: cid, IP: endpoint.IP}) } return http.StatusOK, rv }
func softOOMKill(cid string, rate float64) { logs.Debug("OOM killed", cid[:12]) conn := g.GetRedisConn() defer g.ReleaseRedisConn(conn) key := fmt.Sprintf("eru:agent:%s:container:reason", cid) if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil { logs.Info("OOM killed set flag", err) } if err := g.Docker.StopContainer(cid, 10); err != nil { logs.Info("OOM killed failed", cid[:12]) return } logs.Info("OOM killed success", cid[:12]) }
func Limit() { if g.Config.Limit.Memory != 0 { logs.Info("App memory soft limit start") isLimit = true go calcMemoryUsage() } }
func HTTPServe() { restfulAPIServer := pat.New() handlers := map[string]map[string]func(*Request) (int, interface{}){ "GET": { "/profile/": profile, "/version/": version, "/api/app/list/": listEruApps, }, "POST": { "/api/container/add/": addNewContainer, "/api/container/:container_id/addvlan/": addVlanForContainer, "/api/container/:container_id/addcalico/": addCalicoForContainer, "/api/container/:container_id/setroute/": setRouteForContainer, "/api/container/:container_id/addroute/": addRouteForContainer, "/api/eip/bind/": bindEIP, "/api/eip/release/": releaseEIP, "/api/container/publish/": publishContainer, "/api/container/unpublish/": unpublishContainer, }, } for method, routes := range handlers { for route, handler := range routes { restfulAPIServer.Add(method, route, http.HandlerFunc(JSONWrapper(handler))) } } http.Handle("/", restfulAPIServer) logs.Info("API http server start at", g.Config.API.Addr) err := http.ListenAndServe(g.Config.API.Addr, nil) if err != nil { logs.Assert(err, "ListenAndServe: ") } }
func SetDefaultRoute(cid, gateway string) bool { lock.Lock() defer lock.Unlock() logs.Info("Set", cid[:12], "default route", gateway) container, err := g.Docker.InspectContainer(cid) if err != nil { logs.Info("RouteSetter inspect docker failed", err) return false } pid := container.State.Pid return setDefaultRoute(cid, gateway, pid) }
func AddRoute(cid, CIDR, ifc string) bool { lock.Lock() defer lock.Unlock() logs.Info("Add", cid[:12], "route", CIDR, ifc) container, err := g.Docker.InspectContainer(cid) if err != nil { logs.Info("RouteSetter inspect docker failed", err) return false } pid := container.State.Pid return addRoute(cid, CIDR, ifc, pid) }
// URL /api/eip/release/ func releaseEIP(req *Request) (int, interface{}) { type EIP struct { ID int `json:"id"` } type Result struct { Succ int `json:"succ"` Err string `json:"err"` Veth string `json:"id"` } eips := []EIP{} decoder := json.NewDecoder(req.Body) if err := decoder.Decode(&eips); err != nil { return http.StatusBadRequest, JSON{"message": "wrong JSON format"} } rv := []Result{} for _, eip := range eips { vethName := fmt.Sprintf("%s%d", common.VLAN_PREFIX, eip.ID) if err := network.DelMacVlanDevice(vethName); err != nil { rv = append(rv, Result{Succ: 0, Veth: vethName, Err: err.Error()}) logs.Info("Release EIP failed", err, vethName) continue } rv = append(rv, Result{Succ: 1, Veth: vethName}) } return http.StatusOK, rv }
func Marshal(obj interface{}) []byte { bytes, err := json.MarshalIndent(obj, "", " ") if err != nil { logs.Info("Utils Marshal:", err) } return bytes }
func main() { g.LoadConfig() g.InitialConn() g.InitTransfers() defer g.CloseConn() lenz.InitLenz() status.InitStatus() network.InitVlan() defer lenz.CloseLenz() utils.WritePid(g.Config.PidFile) defer os.Remove(g.Config.PidFile) app.Limit() app.Metric() api.Serve() status.Start() health.Check() var c = make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) signal.Notify(c, syscall.SIGTERM) signal.Notify(c, syscall.SIGHUP) signal.Notify(c, syscall.SIGKILL) signal.Notify(c, syscall.SIGQUIT) logs.Info("Eru Agent Catch", <-c) }
func InitTransfers() { Transfers = consistent.New() for _, transfer := range Config.Metrics.Transfers { Transfers.Add(transfer) } logs.Info("Transfers initiated") }
func InitVlan() { Devices = consistent.New() for _, device := range g.Config.VLan.Physical { Devices.Add(device) } logs.Info("Vlan initiated") }
func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp { name, entrypoint, ident := utils.GetAppInfo(container.Name) if name == "" { logs.Info("Container name invaild", container.Name) return nil } logs.Debug("Eru App", name, entrypoint, ident) transfer, _ := g.Transfers.Get(container.ID, 0) client := falcon.CreateFalconClient( transfer, time.Duration(g.Config.Metrics.Timeout)*time.Millisecond, ) step := time.Duration(g.Config.Metrics.Step) * time.Second extend["hostname"] = g.Config.HostName extend["cid"] = container.ID[:12] extend["ident"] = ident tag := []string{} for k, v := range extend { tag = append(tag, fmt.Sprintf("%s=%v", k, v)) } endpoint := fmt.Sprintf("%s-%s", name, entrypoint) meta := defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend} metric := metric.CreateMetric(step, client, strings.Join(tag, ","), endpoint) eruApp := &EruApp{meta, metric} return eruApp }
// URL /api/eip/bind/ func bindEIP(req *Request) (int, interface{}) { type EIP struct { ID int `json:"id"` IP string `json:"ip"` Broadcast string `json:"broadcast"` } type Result struct { Succ int `json:"succ"` Err string `json:"err"` IP string `json:"ip"` } eips := []EIP{} decoder := json.NewDecoder(req.Body) if err := decoder.Decode(&eips); err != nil { return http.StatusBadRequest, JSON{"message": "wrong JSON format"} } rv := []Result{} for _, eip := range eips { vethName := fmt.Sprintf("%s%d", common.VLAN_PREFIX, eip.ID) veth, err := network.AddMacVlanDevice(vethName, vethName) if err != nil { rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()}) logs.Info("API add EIP failed", err) continue } if err := network.BindAndSetup(veth, eip.IP); err != nil { rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()}) network.DelVlan(veth) logs.Info("API bind EIP failed", err) continue } if err := network.SetBroadcast(vethName, eip.Broadcast); err != nil { rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()}) network.DelVlan(veth) logs.Info("API set broadcast failed", err) continue } rv = append(rv, Result{Succ: 1, IP: eip.IP}) } return http.StatusOK, rv }
func Metric() { metric.SetGlobalSetting( g.Docker, time.Duration(common.STATS_TIMEOUT), time.Duration(common.STATS_FORCE_DONE), common.VLAN_PREFIX, common.DEFAULT_BR, ) logs.Info("Metrics initiated") }
func (self *UpStream) createUDPConn() error { self.scheme = "udp" udpAddr, err := net.ResolveUDPAddr("udp", self.addr) if err != nil { logs.Info("Resolve", self.addr, "failed", err) return err } conn, err := net.DialUDP("udp", nil, udpAddr) if err != nil { logs.Info("Connect backend failed", err) return err } self.conn = conn self.encoder = json.NewEncoder(conn) self.Close = conn.Close return nil }
func DelMacVlanDevice(vethName string) error { logs.Info("Release macvlan device", vethName) link, err := netlink.LinkByName(vethName) if err != nil { return err } DelVlan(link) return nil }
func LoadConfig() { var configPath string var version bool flag.BoolVar(&logs.Mode, "DEBUG", false, "enable debug") flag.StringVar(&configPath, "c", "agent.yaml", "config file") flag.BoolVar(&version, "v", false, "show version") flag.Parse() if version { logs.Info("Version", common.VERSION) os.Exit(0) } load(configPath) }