func monitor() {
for event := range events {
switch event.Status {
case common.STATUS_DIE:
logs.Debug("Status", event.Status, event.ID[:12], event.From)
app.Remove(event.ID)
reportContainerDeath(event.ID)
case common.STATUS_START:
logs.Debug("Status", event.Status, event.ID[:12], event.From)
// if not in watching list, just ignore it
if meta := getContainerMeta(event.ID); meta != nil && !app.Valid(event.ID) {
container, err := g.Docker.InspectContainer(event.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
break
}
eruApp := app.NewEruApp(container, meta)
if eruApp == nil {
logs.Info("Create EruApp failed")
break
}
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(event.ID)
}
}
}
}
func ListenAndServe() error {
svrConfig := InitSSHConfig()
listener, err := net.Listen("tcp", g.Config.Bind)
if err != nil {
logs.Assert(err, "net.Listen failed")
}
defer listener.Close()
for {
conn, err := listener.Accept()
if err != nil {
logs.Assert(err, "listen.Accept failed")
}
sshConn := &SSHConn{conn, svrConfig}
go func() {
if err := sshConn.serve(); err != nil {
logs.Info("Error occured while serving", err)
return
}
logs.Info("Connection closed.")
}()
}
}
func setDefaultRoute(cid, gateway string, pid int) bool {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
origins, err := netns.Get()
if err != nil {
logs.Info("Get orignal namespace failed", err)
return false
}
defer origins.Close()
ns, err := netns.GetFromPid(pid)
if err != nil {
logs.Info("Get container namespace failed", err)
return false
}
netns.Set(ns)
defer ns.Close()
defer netns.Set(origins)
if err := delDefaultRoute(); err != nil {
logs.Info("Delete default routing table failed", err)
return false
}
if err := addDefaultRoute(gateway); err != nil {
logs.Info("Add default route failed", err)
return false
}
logs.Info("Set default route success", cid[:12], gateway)
return true
}
func getContainerMeta(cid string) map[string]interface{} {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn)
if err != nil {
logs.Info("Status get meta", err)
return nil
}
var result map[string]interface{}
if rep.IsNil() {
return nil
}
if b, err := rep.Bytes(); err != nil {
logs.Info("Status get meta", err)
return nil
} else {
if err := json.Unmarshal(b, &result); err != nil {
logs.Info("Status unmarshal meta", err)
return nil
}
}
return result
}
// URL /api/container/add/
func addNewContainer(req *Request) (int, interface{}) {
type Data struct {
Control string `json:"control"`
ContainerID string `json:"container_id"`
Meta map[string]interface{} `json:"meta"`
}
data := &Data{}
decoder := json.NewDecoder(req.Body)
err := decoder.Decode(data)
if err != nil {
return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
}
switch data.Control {
case "+":
if app.Valid(data.ContainerID) {
break
}
logs.Info("API status watch", data.ContainerID)
container, err := g.Docker.InspectContainer(data.ContainerID)
if err != nil {
logs.Info("API status inspect docker failed", err)
break
}
if eruApp := app.NewEruApp(container, data.Meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
}
}
return http.StatusOK, JSON{"message": "ok"}
}
func addRoute(cid, CIDR, ifc string, pid int) bool {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
origins, err := netns.Get()
if err != nil {
logs.Info("Get orignal namespace failed", err)
return false
}
defer origins.Close()
ns, err := netns.GetFromPid(pid)
if err != nil {
logs.Info("Get container namespace failed", err)
return false
}
netns.Set(ns)
defer ns.Close()
defer netns.Set(origins)
if err := addRouteByLink(CIDR, ifc); err != nil {
logs.Info("Add route failed", err)
return false
}
logs.Info("Add route success", cid[:12], CIDR, ifc)
return true
}
func (self *EruApp) Report() {
t := time.NewTicker(self.Step)
defer t.Stop()
defer self.Client.Close()
defer logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report stop")
logs.Info(self.Name, self.EntryPoint, self.ID[:12], "metrics report start")
for {
select {
case now := <-t.C:
go func() {
if info, err := self.UpdateStats(self.ID); err == nil {
if isLimit {
limitChan <- SoftLimit{self.ID, info}
}
rate := self.CalcRate(info, now)
self.SaveLast(info)
go self.Send(rate)
} else {
logs.Info("Update mertic failed", self.ID[:12])
}
}()
case <-self.Stop:
return
}
}
}
func setUpVLan(cid, ips string, pid int, veth netlink.Link) bool {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
origns, err := netns.Get()
if err != nil {
logs.Info("Get orignal namespace failed", err)
return false
}
defer origns.Close()
ns, err := netns.GetFromPid(pid)
if err != nil {
logs.Info("Get container namespace failed", err)
return false
}
netns.Set(ns)
defer ns.Close()
defer netns.Set(origns)
if err := BindAndSetup(veth, ips); err != nil {
logs.Info("Bind and setup NIC failed", err)
DelVlan(veth)
return false
}
logs.Info("Add vlan device success", cid[:12])
return true
}
func SetDefaultRoute(cid, gateway string) bool {
_, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("VLanSetter inspect docker failed", err)
return false
}
logs.Info("Set default route success", cid, gateway)
return true
}
func AddRoute(cid, CIDR string, ifc string) bool {
_, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("VLanSetter inspect docker failed", err)
return false
}
logs.Info("Add route success", cid, CIDR, ifc)
return true
}
func AddVlan(vethName, ips, cid string) bool {
_, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("VLanSetter inspect docker failed", err)
return false
}
logs.Info("Add VLAN device success", cid, vethName)
return true
}
func load() {
containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
logs.Assert(err, "List containers")
}
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
logs.Debug("Status get targets from", containersKey)
rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
if err != nil {
logs.Assert(err, "Status get targets")
}
if rep.IsNil() {
return
}
targets, err := rep.Map()
if err != nil {
logs.Assert(err, "Status load targets")
}
logs.Debug("Status targets:", targets)
logs.Info("Status load container")
for _, container := range containers {
if _, ok := targets[container.ID]; !ok {
continue
}
status := getStatus(container.Status)
if status != common.STATUS_START {
reportContainerDeath(container.ID)
continue
}
var meta map[string]interface{}
if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
logs.Info("Status load failed", err)
continue
}
c, err := g.Docker.InspectContainer(container.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
continue
}
if eruApp := app.NewEruApp(c, meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(container.ID)
}
}
}
func CloseLenz() {
logs.Info("Close all lenz streamer")
routes, err := Router.GetAll()
if err != nil {
logs.Info("Get all lenz route failed", err)
return
}
for _, route := range routes {
if !Router.Remove(route.ID) {
logs.Info("Close lenz route failed", route.ID)
}
}
}
// URL /api/container/:container_id/addcalico/
func addCalicoForContainer(req *Request) (int, interface{}) {
type Endpoint struct {
Nid int `json:"nid"`
Profile string `json:"profile"`
IP string `json:"ip"`
Append bool `json:"append"`
}
type Result struct {
Succ int `json:"succ"`
ContainerID string `json:"container_id"`
IP string `json:"ip"`
Err string `json:"err"`
}
if g.Config.VLan.Calico == "" {
return http.StatusBadRequest, JSON{"message": "Agent not enable calico support"}
}
cid := req.URL.Query().Get(":container_id")
env := os.Environ()
env = append(env, fmt.Sprintf("ETCD_AUTHORITY=%s", g.Config.VLan.Calico))
endpoints := []Endpoint{}
decoder := json.NewDecoder(req.Body)
if err := decoder.Decode(&endpoints); err != nil {
return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
}
rv := []Result{}
for seq, endpoint := range endpoints {
vethName := fmt.Sprintf("%s%d.%d", common.VLAN_PREFIX, endpoint.Nid, seq)
if err := network.AddCalico(env, endpoint.Append, cid, vethName, endpoint.IP); err != nil {
rv = append(rv, Result{Succ: 0, ContainerID: cid, IP: endpoint.IP, Err: err.Error()})
logs.Info("API calico add interface failed", err)
continue
}
//TODO remove when eru-core support ACL
// currently only one profile is used
if err := network.BindCalicoProfile(env, cid, endpoint.Profile); err != nil {
rv = append(rv, Result{Succ: 0, ContainerID: cid, IP: endpoint.IP, Err: err.Error()})
logs.Info("API calico add profile failed", err)
continue
}
rv = append(rv, Result{Succ: 1, ContainerID: cid, IP: endpoint.IP})
}
return http.StatusOK, rv
}
func softOOMKill(cid string, rate float64) {
logs.Debug("OOM killed", cid[:12])
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
logs.Info("OOM killed set flag", err)
}
if err := g.Docker.StopContainer(cid, 10); err != nil {
logs.Info("OOM killed failed", cid[:12])
return
}
logs.Info("OOM killed success", cid[:12])
}
func Limit() {
if g.Config.Limit.Memory != 0 {
logs.Info("App memory soft limit start")
isLimit = true
go calcMemoryUsage()
}
}
func HTTPServe() {
restfulAPIServer := pat.New()
handlers := map[string]map[string]func(*Request) (int, interface{}){
"GET": {
"/profile/": profile,
"/version/": version,
"/api/app/list/": listEruApps,
},
"POST": {
"/api/container/add/": addNewContainer,
"/api/container/:container_id/addvlan/": addVlanForContainer,
"/api/container/:container_id/addcalico/": addCalicoForContainer,
"/api/container/:container_id/setroute/": setRouteForContainer,
"/api/container/:container_id/addroute/": addRouteForContainer,
"/api/eip/bind/": bindEIP,
"/api/eip/release/": releaseEIP,
"/api/container/publish/": publishContainer,
"/api/container/unpublish/": unpublishContainer,
},
}
for method, routes := range handlers {
for route, handler := range routes {
restfulAPIServer.Add(method, route, http.HandlerFunc(JSONWrapper(handler)))
}
}
http.Handle("/", restfulAPIServer)
logs.Info("API http server start at", g.Config.API.Addr)
err := http.ListenAndServe(g.Config.API.Addr, nil)
if err != nil {
logs.Assert(err, "ListenAndServe: ")
}
}
func SetDefaultRoute(cid, gateway string) bool {
lock.Lock()
defer lock.Unlock()
logs.Info("Set", cid[:12], "default route", gateway)
container, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("RouteSetter inspect docker failed", err)
return false
}
pid := container.State.Pid
return setDefaultRoute(cid, gateway, pid)
}
func AddRoute(cid, CIDR, ifc string) bool {
lock.Lock()
defer lock.Unlock()
logs.Info("Add", cid[:12], "route", CIDR, ifc)
container, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("RouteSetter inspect docker failed", err)
return false
}
pid := container.State.Pid
return addRoute(cid, CIDR, ifc, pid)
}
// URL /api/eip/release/
func releaseEIP(req *Request) (int, interface{}) {
type EIP struct {
ID int `json:"id"`
}
type Result struct {
Succ int `json:"succ"`
Err string `json:"err"`
Veth string `json:"id"`
}
eips := []EIP{}
decoder := json.NewDecoder(req.Body)
if err := decoder.Decode(&eips); err != nil {
return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
}
rv := []Result{}
for _, eip := range eips {
vethName := fmt.Sprintf("%s%d", common.VLAN_PREFIX, eip.ID)
if err := network.DelMacVlanDevice(vethName); err != nil {
rv = append(rv, Result{Succ: 0, Veth: vethName, Err: err.Error()})
logs.Info("Release EIP failed", err, vethName)
continue
}
rv = append(rv, Result{Succ: 1, Veth: vethName})
}
return http.StatusOK, rv
}
func Marshal(obj interface{}) []byte {
bytes, err := json.MarshalIndent(obj, "", " ")
if err != nil {
logs.Info("Utils Marshal:", err)
}
return bytes
}
func main() {
g.LoadConfig()
g.InitialConn()
g.InitTransfers()
defer g.CloseConn()
lenz.InitLenz()
status.InitStatus()
network.InitVlan()
defer lenz.CloseLenz()
utils.WritePid(g.Config.PidFile)
defer os.Remove(g.Config.PidFile)
app.Limit()
app.Metric()
api.Serve()
status.Start()
health.Check()
var c = make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
signal.Notify(c, syscall.SIGTERM)
signal.Notify(c, syscall.SIGHUP)
signal.Notify(c, syscall.SIGKILL)
signal.Notify(c, syscall.SIGQUIT)
logs.Info("Eru Agent Catch", <-c)
}
func InitTransfers() {
Transfers = consistent.New()
for _, transfer := range Config.Metrics.Transfers {
Transfers.Add(transfer)
}
logs.Info("Transfers initiated")
}
func InitVlan() {
Devices = consistent.New()
for _, device := range g.Config.VLan.Physical {
Devices.Add(device)
}
logs.Info("Vlan initiated")
}
func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp {
name, entrypoint, ident := utils.GetAppInfo(container.Name)
if name == "" {
logs.Info("Container name invaild", container.Name)
return nil
}
logs.Debug("Eru App", name, entrypoint, ident)
transfer, _ := g.Transfers.Get(container.ID, 0)
client := falcon.CreateFalconClient(
transfer,
time.Duration(g.Config.Metrics.Timeout)*time.Millisecond,
)
step := time.Duration(g.Config.Metrics.Step) * time.Second
extend["hostname"] = g.Config.HostName
extend["cid"] = container.ID[:12]
extend["ident"] = ident
tag := []string{}
for k, v := range extend {
tag = append(tag, fmt.Sprintf("%s=%v", k, v))
}
endpoint := fmt.Sprintf("%s-%s", name, entrypoint)
meta := defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend}
metric := metric.CreateMetric(step, client, strings.Join(tag, ","), endpoint)
eruApp := &EruApp{meta, metric}
return eruApp
}
// URL /api/eip/bind/
func bindEIP(req *Request) (int, interface{}) {
type EIP struct {
ID int `json:"id"`
IP string `json:"ip"`
Broadcast string `json:"broadcast"`
}
type Result struct {
Succ int `json:"succ"`
Err string `json:"err"`
IP string `json:"ip"`
}
eips := []EIP{}
decoder := json.NewDecoder(req.Body)
if err := decoder.Decode(&eips); err != nil {
return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
}
rv := []Result{}
for _, eip := range eips {
vethName := fmt.Sprintf("%s%d", common.VLAN_PREFIX, eip.ID)
veth, err := network.AddMacVlanDevice(vethName, vethName)
if err != nil {
rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()})
logs.Info("API add EIP failed", err)
continue
}
if err := network.BindAndSetup(veth, eip.IP); err != nil {
rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()})
network.DelVlan(veth)
logs.Info("API bind EIP failed", err)
continue
}
if err := network.SetBroadcast(vethName, eip.Broadcast); err != nil {
rv = append(rv, Result{Succ: 0, IP: eip.IP, Err: err.Error()})
network.DelVlan(veth)
logs.Info("API set broadcast failed", err)
continue
}
rv = append(rv, Result{Succ: 1, IP: eip.IP})
}
return http.StatusOK, rv
}
func Metric() {
metric.SetGlobalSetting(
g.Docker, time.Duration(common.STATS_TIMEOUT),
time.Duration(common.STATS_FORCE_DONE),
common.VLAN_PREFIX, common.DEFAULT_BR,
)
logs.Info("Metrics initiated")
}
func (self *UpStream) createUDPConn() error {
self.scheme = "udp"
udpAddr, err := net.ResolveUDPAddr("udp", self.addr)
if err != nil {
logs.Info("Resolve", self.addr, "failed", err)
return err
}
conn, err := net.DialUDP("udp", nil, udpAddr)
if err != nil {
logs.Info("Connect backend failed", err)
return err
}
self.conn = conn
self.encoder = json.NewEncoder(conn)
self.Close = conn.Close
return nil
}
func DelMacVlanDevice(vethName string) error {
logs.Info("Release macvlan device", vethName)
link, err := netlink.LinkByName(vethName)
if err != nil {
return err
}
DelVlan(link)
return nil
}
func LoadConfig() {
var configPath string
var version bool
flag.BoolVar(&logs.Mode, "DEBUG", false, "enable debug")
flag.StringVar(&configPath, "c", "agent.yaml", "config file")
flag.BoolVar(&version, "v", false, "show version")
flag.Parse()
if version {
logs.Info("Version", common.VERSION)
os.Exit(0)
}
load(configPath)
}