func routeWatcher() {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
subs := gore.NewSubscriptions(conn)
defer subs.Close()
subKey := fmt.Sprintf("eru:agent:%s:route", g.Config.HostName)
logs.Debug("API route subscribe", subKey)
subs.Subscribe(subKey)
for message := range subs.Message() {
if message == nil {
logs.Info("API route watcher shutdown")
break
}
command := string(message.Message)
logs.Debug("API route watcher get", command)
parser := strings.Split(command, "|")
if len(parser) != 2 {
logs.Info("API route watcher command invaild", command)
continue
}
cid, gateway := parser[0], parser[1]
if !network.SetDefaultRoute(cid, gateway) {
logs.Info("Set default route failed")
}
}
}
func getContainerMeta(cid string) map[string]interface{} {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn)
if err != nil {
logs.Info("Status get meta", err)
return nil
}
var result map[string]interface{}
if rep.IsNil() {
return nil
}
if b, err := rep.Bytes(); err != nil {
logs.Info("Status get meta", err)
return nil
} else {
if err := json.Unmarshal(b, &result); err != nil {
logs.Info("Status unmarshal meta", err)
return nil
}
}
return result
}
func load() {
containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
logs.Assert(err, "List containers")
}
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
logs.Debug("Status get targets from", containersKey)
rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
if err != nil {
logs.Assert(err, "Status get targets")
}
if rep.IsNil() {
return
}
targets, err := rep.Map()
if err != nil {
logs.Assert(err, "Status load targets")
}
logs.Debug("Status targets:", targets)
logs.Info("Status load container")
for _, container := range containers {
if _, ok := targets[container.ID]; !ok {
continue
}
status := getStatus(container.Status)
if status != common.STATUS_START {
reportContainerDeath(container.ID)
continue
}
var meta map[string]interface{}
if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
logs.Info("Status load failed", err)
continue
}
c, err := g.Docker.InspectContainer(container.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
continue
}
if eruApp := app.NewEruApp(c, meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(container.ID)
}
}
}
func vlanWatcher() {
conn := g.GetRedisConn()
report := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
defer g.ReleaseRedisConn(report)
subs := gore.NewSubscriptions(conn)
defer subs.Close()
subKey := fmt.Sprintf("eru:agent:%s:vlan", g.Config.HostName)
logs.Debug("API vlan subscribe", subKey)
subs.Subscribe(subKey)
for message := range subs.Message() {
if message == nil {
logs.Info("API vLan watcher shutdown")
break
}
command := string(message.Message)
logs.Debug("API vlan watcher get", command)
parser := strings.Split(command, "|")
if len(parser) <= 2 {
logs.Info("API vlan watcher command invaild", command)
continue
}
taskID, cid := parser[0], parser[1]
feedKey := fmt.Sprintf("eru:agent:%s:feedback", taskID)
for seq, content := range parser[2:] {
p := strings.Split(content, ":")
if len(p) != 2 {
logs.Info("API vlan watcher ips invaild", content)
continue
}
nid, ips := p[0], p[1]
vethName := fmt.Sprintf("%s%s.%d", common.VLAN_PREFIX, nid, seq)
if network.AddVLan(vethName, ips, cid) {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("1|%s|%s|%s", cid, vethName, ips)).Run(report)
continue
}
gore.NewCommand("LPUSH", feedKey, "0|||").Run(report)
}
}
}
func softOOMKill(cid string, rate float64) {
logs.Debug("OOM killed", cid[:12])
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
logs.Info("OOM killed set flag", err)
}
if err := g.Docker.StopContainer(cid, 10); err != nil {
logs.Info("OOM killed failed", cid[:12])
return
}
logs.Info("OOM killed success", cid[:12])
}
func statusWatcher() {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
subs := gore.NewSubscriptions(conn)
defer subs.Close()
subKey := fmt.Sprintf("eru:agent:%s:watcher", g.Config.HostName)
logs.Debug("API status subscribe", subKey)
subs.Subscribe(subKey)
for message := range subs.Message() {
if message == nil {
logs.Info("API status watcher shutdown")
break
}
command := string(message.Message)
logs.Debug("API status watcher get", command)
parser := strings.Split(command, "|")
if len(parser) != 3 {
logs.Info("API status watcher command invaild", command)
continue
}
control, cid, metaString := parser[0], parser[1], parser[2]
switch control {
case "+":
if app.Valid(cid) {
break
}
logs.Info("API status watch", cid[:12])
container, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("API status inspect docker failed", err)
break
}
var meta map[string]interface{}
if err := json.Unmarshal([]byte(metaString), &meta); err != nil {
logs.Info("API status load failed", err)
break
}
if eruApp := app.NewEruApp(container, meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
}
}
}
}
func reportContainerDeath(cid string) {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
flagKey := fmt.Sprintf("eru:agent:%s:container:flag", cid)
rep, err := gore.NewCommand("GET", flagKey).Run(conn)
if err != nil {
logs.Info("Status failed in get flag", err)
return
}
if !rep.IsNil() {
gore.NewCommand("DEL", flagKey).Run(conn)
logs.Debug(cid[:12], "Status flag set, ignore")
return
}
url := fmt.Sprintf("%s/api/container/%s/kill", g.Config.Eru.Endpoint, cid)
utils.DoPut(url)
logs.Debug(cid[:12], "dead, remove from watching list")
}