func getContainerMeta(cid string) map[string]interface{} {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn)
if err != nil {
logs.Info("Status get meta", err)
return nil
}
var result map[string]interface{}
if rep.IsNil() {
return nil
}
if b, err := rep.Bytes(); err != nil {
logs.Info("Status get meta", err)
return nil
} else {
if err := json.Unmarshal(b, &result); err != nil {
logs.Info("Status unmarshal meta", err)
return nil
}
}
return result
}
func load() {
containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
logs.Assert(err, "List containers")
}
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
logs.Debug("Status get targets from", containersKey)
rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
if err != nil {
logs.Assert(err, "Status get targets")
}
if rep.IsNil() {
return
}
targets, err := rep.Map()
if err != nil {
logs.Assert(err, "Status load targets")
}
logs.Debug("Status targets:", targets)
logs.Info("Status load container")
for _, container := range containers {
if _, ok := targets[container.ID]; !ok {
continue
}
status := getStatus(container.Status)
if status != common.STATUS_START {
reportContainerDeath(container.ID)
continue
}
var meta map[string]interface{}
if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
logs.Info("Status load failed", err)
continue
}
c, err := g.Docker.InspectContainer(container.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
continue
}
if eruApp := app.NewEruApp(c, meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(container.ID)
}
}
}
func softOOMKill(cid string, rate float64) {
logs.Debug("OOM killed", cid[:12])
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
logs.Info("OOM killed set flag", err)
}
if err := g.Docker.StopContainer(cid, 10); err != nil {
logs.Info("OOM killed failed", cid[:12])
return
}
logs.Info("OOM killed success", cid[:12])
}
func reportContainerDeath(cid string) {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
flagKey := fmt.Sprintf("eru:agent:%s:container:flag", cid)
rep, err := gore.NewCommand("GET", flagKey).Run(conn)
if err != nil {
logs.Info("Status failed in get flag", err)
return
}
if !rep.IsNil() {
gore.NewCommand("DEL", flagKey).Run(conn)
logs.Debug(cid[:12], "Status flag set, ignore")
return
}
url := fmt.Sprintf("%s/api/container/%s/kill/", g.Config.Eru.Endpoint, cid)
utils.DoPut(url)
logs.Debug(cid[:12], "dead, remove from watching list")
}