// URL /api/container/:container_id/addvlan/
func addVlanForContainer(req *Request) (int, interface{}) {
type IP struct {
Nid int `json:"nid"`
IP string `json:"ip"`
}
type Data struct {
TaskID int `json:"task_id"`
IPs []IP `json:"ips"`
}
conn := g.GetRedisConn()
defer g.Rds.Release(conn)
cid := req.URL.Query().Get(":container_id")
data := &Data{}
decoder := json.NewDecoder(req.Body)
err := decoder.Decode(data)
if err != nil {
return http.StatusBadRequest, JSON{"message": "wrong JSON format"}
}
feedKey := fmt.Sprintf("eru:agent:%s:feedback", data.TaskID)
for seq, ip := range data.IPs {
vethName := fmt.Sprintf("%s%d.%d", common.VLAN_PREFIX, ip.Nid, seq)
if network.AddVLan(vethName, ip.IP, cid) {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("1|%s|%s|%s", cid, vethName, ip.IP)).Run(conn)
continue
} else {
gore.NewCommand("LPUSH", feedKey, "0|||").Run(conn)
}
}
return http.StatusOK, JSON{"message": "ok"}
}
func getContainerMeta(cid string) map[string]interface{} {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
rep, err := gore.NewCommand("HGET", containersKey, cid).Run(conn)
if err != nil {
logs.Info("Status get meta", err)
return nil
}
var result map[string]interface{}
if rep.IsNil() {
return nil
}
if b, err := rep.Bytes(); err != nil {
logs.Info("Status get meta", err)
return nil
} else {
if err := json.Unmarshal(b, &result); err != nil {
logs.Info("Status unmarshal meta", err)
return nil
}
}
return result
}
func load() {
containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
logs.Assert(err, "List containers")
}
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
logs.Debug("Status get targets from", containersKey)
rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
if err != nil {
logs.Assert(err, "Status get targets")
}
if rep.IsNil() {
return
}
targets, err := rep.Map()
if err != nil {
logs.Assert(err, "Status load targets")
}
logs.Debug("Status targets:", targets)
logs.Info("Status load container")
for _, container := range containers {
if _, ok := targets[container.ID]; !ok {
continue
}
status := getStatus(container.Status)
if status != common.STATUS_START {
reportContainerDeath(container.ID)
continue
}
var meta map[string]interface{}
if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
logs.Info("Status load failed", err)
continue
}
c, err := g.Docker.InspectContainer(container.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
continue
}
if eruApp := app.NewEruApp(c, meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(container.ID)
}
}
}
func vlanWatcher() {
conn := g.GetRedisConn()
report := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
defer g.ReleaseRedisConn(report)
subs := gore.NewSubscriptions(conn)
defer subs.Close()
subKey := fmt.Sprintf("eru:agent:%s:vlan", g.Config.HostName)
logs.Debug("API vlan subscribe", subKey)
subs.Subscribe(subKey)
for message := range subs.Message() {
if message == nil {
logs.Info("API vLan watcher shutdown")
break
}
command := string(message.Message)
logs.Debug("API vlan watcher get", command)
parser := strings.Split(command, "|")
if len(parser) <= 2 {
logs.Info("API vlan watcher command invaild", command)
continue
}
taskID, cid := parser[0], parser[1]
feedKey := fmt.Sprintf("eru:agent:%s:feedback", taskID)
for seq, content := range parser[2:] {
p := strings.Split(content, ":")
if len(p) != 2 {
logs.Info("API vlan watcher ips invaild", content)
continue
}
nid, ips := p[0], p[1]
vethName := fmt.Sprintf("%s%s.%d", common.VLAN_PREFIX, nid, seq)
if network.AddVLan(vethName, ips, cid) {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("1|%s|%s|%s", cid, vethName, ips)).Run(report)
continue
}
gore.NewCommand("LPUSH", feedKey, "0|||").Run(report)
}
}
}
func reportContainerDeath(cid string) {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
flagKey := fmt.Sprintf("eru:agent:%s:container:flag", cid)
rep, err := gore.NewCommand("GET", flagKey).Run(conn)
if err != nil {
logs.Info("Status failed in get flag", err)
return
}
if !rep.IsNil() {
gore.NewCommand("DEL", flagKey).Run(conn)
logs.Debug(cid[:12], "Status flag set, ignore")
return
}
url := fmt.Sprintf("%s/api/container/%s/kill/", g.Config.Eru.Endpoint, cid)
utils.DoPut(url)
logs.Debug(cid[:12], "dead, remove from watching list")
}
func (self *VLanSetter) addVLan(feedKey, content, ident, containerID string) {
conn, err := common.Rds.Acquire()
if err != nil || conn == nil {
logs.Info(err, "Get redis conn")
return
}
defer common.Rds.Release(conn)
parser := strings.Split(content, ":")
if len(parser) != 2 {
logs.Info("Seq and Ips Invaild", content)
return
}
seq, ips := parser[0], parser[1]
device, _ := self.Devices.Get(ident, 0)
vethName := fmt.Sprintf("%s%s.%s", common.VLAN_PREFIX, ident, seq)
logs.Info("Add new VLan to", vethName, containerID)
if err := netlink.NetworkLinkAddMacVlan(device, vethName, "bridge"); err != nil {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("0|||")).Run(conn)
logs.Info("Create macvlan device failed", err)
return
}
container, err := common.Docker.InspectContainer(containerID)
if err != nil {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("0|||")).Run(conn)
logs.Info("VLanSetter inspect docker failed", err)
self.delVLan(vethName)
return
}
ifc, _ := net.InterfaceByName(vethName)
if err := netlink.NetworkSetNsPid(ifc, container.State.Pid); err != nil {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("0|||")).Run(conn)
logs.Info("Set macvlan device into container failed", err)
self.delVLan(vethName)
return
}
pid := strconv.Itoa(container.State.Pid)
cmd := exec.Command("nsenter", "-t", pid, "-n", "ip", "addr", "add", ips, "dev", vethName)
if err := cmd.Run(); err != nil {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("0|||")).Run(conn)
logs.Info("Bind ip in container failed", err)
return
}
cmd = exec.Command("nsenter", "-t", pid, "-n", "ip", "link", "set", vethName, "up")
if err := cmd.Run(); err != nil {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("0|||")).Run(conn)
logs.Info("Set up veth in container failed", err)
return
}
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("1|%s|%s|%s", containerID, vethName, ips)).Run(conn)
logs.Info("Add VLAN device success", containerID, ident)
}
func CheckKey(user, keyHex string) bool {
rds := g.GetRedisConn()
defer g.ReleaseRedisConn(rds)
checkKey := fmt.Sprintf(common.CHECK_KEY, user)
var err error
var rep *gore.Reply
if rep, err = gore.NewCommand("HGET", checkKey, keyHex).Run(rds); err != nil {
logs.Info("Get info failed", err)
return false
}
if rep.IsNil() {
return false
}
return true
}
func isInWatchingSet(cid string) bool {
conn, err := common.Rds.Acquire()
if err != nil || conn == nil {
logs.Assert(err, "Get redis conn")
}
defer common.Rds.Release(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers", config.HostName)
rep, err := gore.NewCommand("SISMEMBER", containersKey, cid).Run(conn)
if err != nil {
logs.Assert(err, "Get targets")
}
repInt, _ := rep.Int()
return repInt == 1
}
func softOOMKill(cid string, rate float64) {
logs.Debug("OOM killed", cid[:12])
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
logs.Info("OOM killed set flag", err)
}
if err := g.Docker.StopContainer(cid, 10); err != nil {
logs.Info("OOM killed failed", cid[:12])
return
}
logs.Info("OOM killed success", cid[:12])
}
func GetRealUserRemote(username string) (string, string) {
var remote string
rds := g.GetRedisConn()
defer g.ReleaseRedisConn(rds)
keys := strings.Split(username, "~")
user, ident := keys[0], keys[1]
routeKey := fmt.Sprintf(common.ROUTE_KEY, user)
var err error
var rep *gore.Reply
if rep, err = gore.NewCommand("HGET", routeKey, ident).Run(rds); err != nil {
logs.Info("Get info failed", err)
return "", ""
}
if rep.IsNil() {
return "", ""
}
remote, _ = rep.String()
return user, remote
}
func (self *StatusMoniter) Load() {
containers, err := common.Docker.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
logs.Assert(err, "List containers")
}
conn, err := common.Rds.Acquire()
if err != nil || conn == nil {
logs.Assert(err, "Get redis conn")
}
defer common.Rds.Release(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers", config.HostName)
logs.Debug("Get targets from", containersKey)
rep, err := gore.NewCommand("SMEMBERS", containersKey).Run(conn)
if err != nil {
logs.Assert(err, "Get targets")
}
targetContainersList := []string{}
rep.Slice(&targetContainersList)
logs.Debug("Targets:", targetContainersList)
targets := map[string]struct{}{}
for _, target := range targetContainersList {
targets[target] = struct{}{}
}
logs.Info("Load container")
for _, container := range containers {
if _, ok := targets[container.ID]; !ok {
continue
}
status := self.getStatus(container.Status)
if status != common.STATUS_START {
reportContainerDeath(container.ID)
continue
}
self.Add(container.ID, container.Names[0])
}
}
func reportContainerDeath(cid string) {
conn, err := common.Rds.Acquire()
if err != nil || conn == nil {
logs.Assert(err, "Get redis conn")
}
defer common.Rds.Release(conn)
rep, err := gore.NewCommand("GET", fmt.Sprintf("eru:agent:%s:container:flag", cid)).Run(conn)
if err != nil {
logs.Assert(err, "failed in GET")
}
if !rep.IsNil() {
logs.Debug(cid, "flag set, ignore")
return
}
url := fmt.Sprintf("%s/api/container/%s/kill", config.Eru.Endpoint, cid)
client := &http.Client{}
req, _ := http.NewRequest("PUT", url, nil)
client.Do(req)
}