func (m *AttachManager) Attach(app *defines.Meta) {
// Not Thread Safe
if m.Attached(app.ID) {
return
}
outrd, outwr := io.Pipe()
errrd, errwr := io.Pipe()
go func() {
err := g.Docker.AttachToContainer(docker.AttachToContainerOptions{
Container: app.ID,
OutputStream: outwr,
ErrorStream: errwr,
Stdin: false,
Stdout: true,
Stderr: true,
Stream: true,
})
outwr.Close()
errwr.Close()
logs.Debug("Lenz Attach", app.ID[:12], "finished")
if err != nil {
logs.Debug("Lenz Attach", app.ID, "failure:", err)
}
m.send(&defines.AttachEvent{Type: "detach", App: app})
m.Lock()
defer m.Unlock()
delete(m.attached, app.ID)
}()
m.Lock()
m.attached[app.ID] = NewLogPump(outrd, errrd, app)
m.Unlock()
m.send(&defines.AttachEvent{Type: "attach", App: app})
logs.Debug("Lenz Attach", app.ID[:12], "success")
}
func routeWatcher() {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
subs := gore.NewSubscriptions(conn)
defer subs.Close()
subKey := fmt.Sprintf("eru:agent:%s:route", g.Config.HostName)
logs.Debug("API route subscribe", subKey)
subs.Subscribe(subKey)
for message := range subs.Message() {
if message == nil {
logs.Info("API route watcher shutdown")
break
}
command := string(message.Message)
logs.Debug("API route watcher get", command)
parser := strings.Split(command, "|")
if len(parser) != 2 {
logs.Info("API route watcher command invaild", command)
continue
}
cid, gateway := parser[0], parser[1]
if !network.SetDefaultRoute(cid, gateway) {
logs.Info("Set default route failed")
}
}
}
func monitor() {
for event := range events {
switch event.Status {
case common.STATUS_DIE:
logs.Debug("Status", event.Status, event.ID[:12], event.From)
app.Remove(event.ID)
reportContainerDeath(event.ID)
case common.STATUS_START:
logs.Debug("Status", event.Status, event.ID[:12], event.From)
// if not in watching list, just ignore it
if meta := getContainerMeta(event.ID); meta != nil && !app.Valid(event.ID) {
container, err := g.Docker.InspectContainer(event.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
break
}
eruApp := app.NewEruApp(container, meta)
if eruApp == nil {
logs.Info("Create EruApp failed")
break
}
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(event.ID)
}
}
}
}
func load() {
containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
logs.Assert(err, "List containers")
}
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
logs.Debug("Status get targets from", containersKey)
rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
if err != nil {
logs.Assert(err, "Status get targets")
}
if rep.IsNil() {
return
}
targets, err := rep.Map()
if err != nil {
logs.Assert(err, "Status load targets")
}
logs.Debug("Status targets:", targets)
logs.Info("Status load container")
for _, container := range containers {
if _, ok := targets[container.ID]; !ok {
continue
}
status := getStatus(container.Status)
if status != common.STATUS_START {
reportContainerDeath(container.ID)
continue
}
var meta map[string]interface{}
if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
logs.Info("Status load failed", err)
continue
}
c, err := g.Docker.InspectContainer(container.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
continue
}
if eruApp := app.NewEruApp(c, meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(container.ID)
}
}
}
func (self *EruApp) send(rate map[string]float64) {
data := []*model.MetricValue{}
for k, d := range rate {
data = append(data, self.newMetricValue(k, d))
}
var resp model.TransferResponse
if err := self.Client.Call("Transfer.Update", data, &resp); err != nil {
logs.Debug("Metrics call Transfer.Update fail", err, self.Name, self.EntryPoint)
return
}
logs.Debug(self.Endpoint, self.Last, &resp)
}
func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp {
name, entrypoint, ident := utils.GetAppInfo(container.Name)
if name == "" {
logs.Info("Container name invaild", container.Name)
return nil
}
logs.Debug("Eru App", name, entrypoint, ident)
transfer, _ := g.Transfers.Get(container.ID, 0)
client := falcon.CreateFalconClient(
transfer,
time.Duration(g.Config.Metrics.Timeout)*time.Millisecond,
)
step := time.Duration(g.Config.Metrics.Step) * time.Second
extend["hostname"] = g.Config.HostName
extend["cid"] = container.ID[:12]
extend["ident"] = ident
tag := []string{}
for k, v := range extend {
tag = append(tag, fmt.Sprintf("%s=%v", k, v))
}
endpoint := fmt.Sprintf("%s-%s", name, entrypoint)
meta := defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend}
metric := metric.CreateMetric(step, client, strings.Join(tag, ","), endpoint)
eruApp := &EruApp{meta, metric}
return eruApp
}
func (self *Metric) getNetStats(result map[string]uint64) (err error) {
s := bufio.NewScanner(self.statFile)
defer self.statFile.Seek(0, 0)
var d uint64
for s.Scan() {
var name string
var n [8]uint64
text := s.Text()
if strings.Index(text, ":") < 1 {
continue
}
ts := strings.Split(text, ":")
fmt.Sscanf(ts[0], "%s", &name)
if !strings.HasPrefix(name, g.vlanPrefix) && name != g.defaultVlan {
continue
}
fmt.Sscanf(ts[1],
"%d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
&n[0], &n[1], &n[2], &n[3], &d, &d, &d, &d,
&n[4], &n[5], &n[6], &n[7], &d, &d, &d, &d,
)
result[name+".inbytes"] = n[0]
result[name+".inpackets"] = n[1]
result[name+".inerrs"] = n[2]
result[name+".indrop"] = n[3]
result[name+".outbytes"] = n[4]
result[name+".outpackets"] = n[5]
result[name+".outerrs"] = n[6]
result[name+".outdrop"] = n[7]
}
logs.Debug("Container net status", result)
return
}
func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp {
name, entrypoint, ident := utils.GetAppInfo(container.Name)
if name == "" {
logs.Info("Container name invaild", container.Name)
return nil
}
logs.Debug("Eru App", name, entrypoint, ident)
transfer, _ := g.Transfers.Get(container.ID, 0)
client := defines.SingleConnRpcClient{
RpcServer: transfer,
Timeout: time.Duration(g.Config.Metrics.Timeout) * time.Millisecond,
}
step := time.Duration(g.Config.Metrics.Step) * time.Second
extend["hostname"] = g.Config.HostName
extend["cid"] = container.ID[:12]
extend["ident"] = ident
tag := []string{}
for k, v := range extend {
tag = append(tag, fmt.Sprintf("%s=%v", k, v))
}
endpoint := fmt.Sprintf("%s-%s", name, entrypoint)
eruApp := &EruApp{
defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend},
defines.Metric{Step: step, Client: client, Tag: strings.Join(tag, ","), Endpoint: endpoint},
nil,
}
eruApp.Stop = make(chan bool)
return eruApp
}
func NewLogPump(stdout, stderr io.Reader, app *defines.Meta) *LogPump {
obj := &LogPump{
app: app,
channels: make(map[chan *defines.Log]struct{}),
}
pump := func(typ string, source io.Reader) {
buf := bufio.NewReader(source)
for {
data, err := buf.ReadBytes('\n')
if err != nil {
if err != io.EOF {
logs.Debug("Lenz Pump:", app.ID, typ, err)
}
return
}
obj.send(&defines.Log{
Data: strings.TrimSuffix(string(data), "\n"),
ID: app.ID,
Name: app.Name,
EntryPoint: app.EntryPoint,
Ident: app.Ident,
Type: typ,
Datetime: time.Now().Format(common.DATETIME_FORMAT),
})
}
}
go pump("stdout", stdout)
go pump("stderr", stderr)
return obj
}
func tcpStreamer(logline *defines.Log, addr string) error {
tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
if err != nil {
logs.Debug("Resolve tcp failed", err)
return err
}
conn, err := net.DialTCP("tcp", nil, tcpAddr)
if err != nil {
logs.Debug("Connect backend failed", err)
return err
}
defer conn.Close()
writeJSON(conn, logline)
return nil
}
func udpStreamer(logline *defines.Log, addr string) error {
udpAddr, err := net.ResolveUDPAddr("udp", addr)
if err != nil {
logs.Debug("Resolve udp failed", err)
return err
}
conn, err := net.DialUDP("udp", nil, udpAddr)
if err != nil {
logs.Debug("Connect backend failed", err)
return err
}
defer conn.Close()
writeJSON(conn, logline)
return nil
}
func GetNetStats(statFile *os.File, result map[string]uint64) (err error) {
s := bufio.NewScanner(statFile)
defer statFile.Seek(0, 0)
var d uint64
for s.Scan() {
var name string
var n [8]uint64
text := s.Text()
if strings.Index(text, ":") < 1 {
continue
}
ts := strings.Split(text, ":")
fmt.Sscanf(ts[0], "%s", &name)
if !strings.HasPrefix(name, common.VLAN_PREFIX) &&
name != common.DEFAULT_BR {
continue
}
fmt.Sscanf(ts[1],
"%d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
&n[0], &n[1], &n[2], &n[3], &d, &d, &d, &d,
&n[4], &n[5], &n[6], &n[7], &d, &d, &d, &d,
)
result[name+".inbytes"] = n[0]
result[name+".inpackets"] = n[1]
result[name+".inerrs"] = n[2]
result[name+".indrop"] = n[3]
result[name+".outbytes"] = n[4]
result[name+".outpackets"] = n[5]
result[name+".outerrs"] = n[6]
result[name+".outdrop"] = n[7]
}
logs.Debug("Container net status", result)
return
}
func statusWatcher() {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
subs := gore.NewSubscriptions(conn)
defer subs.Close()
subKey := fmt.Sprintf("eru:agent:%s:watcher", g.Config.HostName)
logs.Debug("API status subscribe", subKey)
subs.Subscribe(subKey)
for message := range subs.Message() {
if message == nil {
logs.Info("API status watcher shutdown")
break
}
command := string(message.Message)
logs.Debug("API status watcher get", command)
parser := strings.Split(command, "|")
if len(parser) != 3 {
logs.Info("API status watcher command invaild", command)
continue
}
control, cid, metaString := parser[0], parser[1], parser[2]
switch control {
case "+":
if app.Valid(cid) {
break
}
logs.Info("API status watch", cid[:12])
container, err := g.Docker.InspectContainer(cid)
if err != nil {
logs.Info("API status inspect docker failed", err)
break
}
var meta map[string]interface{}
if err := json.Unmarshal([]byte(metaString), &meta); err != nil {
logs.Info("API status load failed", err)
break
}
if eruApp := app.NewEruApp(container, meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
}
}
}
}
func InitLenz() {
Attacher = NewAttachManager(g.Docker)
Router = NewRouteManager(Attacher, g.Config.Lenz.Stdout)
Routefs = RouteFileStore(g.Config.Lenz.Routes)
if len(g.Config.Lenz.Forwards) > 0 {
logs.Debug("Lenz Routing all to", g.Config.Lenz.Forwards)
target := defines.Target{Addrs: g.Config.Lenz.Forwards}
route := defines.Route{ID: "lenz_default", Target: &target}
route.LoadBackends()
Router.Add(&route)
}
if _, err := os.Stat(g.Config.Lenz.Routes); err == nil {
logs.Debug("Loading and persisting routes in", g.Config.Lenz.Routes)
logs.Assert(Router.Load(Routefs), "persistor")
}
logs.Info("Lenz initiated")
}
func GetNetStats(exec *docker.Exec) (result map[string]uint64, err error) {
outr, outw := io.Pipe()
defer outr.Close()
success := make(chan struct{})
failure := make(chan error)
go func() {
// TODO: 防止被err流block, 删掉先, 之后记得补上
err = g.Docker.StartExec(
exec.ID,
docker.StartExecOptions{
OutputStream: outw,
Success: success,
},
)
outw.Close()
if err != nil {
close(success)
failure <- err
}
}()
if _, ok := <-success; ok {
success <- struct{}{}
result = map[string]uint64{}
s := bufio.NewScanner(outr)
var d uint64
for s.Scan() {
var name string
var n [8]uint64
text := s.Text()
if strings.Index(text, ":") < 1 {
continue
}
ts := strings.Split(text, ":")
fmt.Sscanf(ts[0], "%s", &name)
if !strings.HasPrefix(name, common.VLAN_PREFIX) {
continue
}
fmt.Sscanf(ts[1],
"%d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
&n[0], &n[1], &n[2], &n[3], &d, &d, &d, &d,
&n[4], &n[5], &n[6], &n[7], &d, &d, &d, &d,
)
result[name+".inbytes"] = n[0]
result[name+".inpackets"] = n[1]
result[name+".inerrs"] = n[2]
result[name+".indrop"] = n[3]
result[name+".outbytes"] = n[4]
result[name+".outpackets"] = n[5]
result[name+".outerrs"] = n[6]
result[name+".outdrop"] = n[7]
}
logs.Debug("Container net status", result)
return
}
err = <-failure
return nil, err
}
func (m *AttachManager) Attach(app *defines.Meta) {
// Not Thread Safe
if m.Attached(app.ID) {
return
}
success := make(chan struct{})
failure := make(chan error)
outrd, outwr := io.Pipe()
errrd, errwr := io.Pipe()
go func() {
err := m.client.AttachToContainer(docker.AttachToContainerOptions{
Container: app.ID,
OutputStream: outwr,
ErrorStream: errwr,
Stdin: false,
Stdout: true,
Stderr: true,
Stream: true,
Success: success,
})
outwr.Close()
errwr.Close()
logs.Debug("Lenz Attach", app.ID[:12], "finished")
if err != nil {
close(success)
failure <- err
}
m.send(&defines.AttachEvent{Type: "detach", App: app})
m.Lock()
defer m.Unlock()
delete(m.attached, app.ID)
}()
_, ok := <-success
if ok {
m.Lock()
m.attached[app.ID] = NewLogPump(outrd, errrd, app)
m.Unlock()
success <- struct{}{}
m.send(&defines.AttachEvent{Type: "attach", App: app})
logs.Debug("Lenz Attach", app.ID[:12], "success")
return
}
logs.Debug("Lenz Attach", app.ID, "failure:", <-failure)
}
func DoPut(url string) {
req, err := http.NewRequest("PUT", url, nil)
if err != nil {
logs.Debug("Gen request failed", err)
return
}
response, err := httpClient.Do(req)
if err != nil {
logs.Debug("Do request failed", err)
return
}
defer response.Body.Close()
data, err := ioutil.ReadAll(response.Body)
if err != nil {
logs.Debug("Read response failed", err)
return
}
logs.Debug("Response:", string(data))
}
func vlanWatcher() {
conn := g.GetRedisConn()
report := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
defer g.ReleaseRedisConn(report)
subs := gore.NewSubscriptions(conn)
defer subs.Close()
subKey := fmt.Sprintf("eru:agent:%s:vlan", g.Config.HostName)
logs.Debug("API vlan subscribe", subKey)
subs.Subscribe(subKey)
for message := range subs.Message() {
if message == nil {
logs.Info("API vLan watcher shutdown")
break
}
command := string(message.Message)
logs.Debug("API vlan watcher get", command)
parser := strings.Split(command, "|")
if len(parser) <= 2 {
logs.Info("API vlan watcher command invaild", command)
continue
}
taskID, cid := parser[0], parser[1]
feedKey := fmt.Sprintf("eru:agent:%s:feedback", taskID)
for seq, content := range parser[2:] {
p := strings.Split(content, ":")
if len(p) != 2 {
logs.Info("API vlan watcher ips invaild", content)
continue
}
nid, ips := p[0], p[1]
vethName := fmt.Sprintf("%s%s.%d", common.VLAN_PREFIX, nid, seq)
if network.AddVLan(vethName, ips, cid) {
gore.NewCommand("LPUSH", feedKey, fmt.Sprintf("1|%s|%s|%s", cid, vethName, ips)).Run(report)
continue
}
gore.NewCommand("LPUSH", feedKey, "0|||").Run(report)
}
}
}
func reportContainerDeath(cid string) {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
flagKey := fmt.Sprintf("eru:agent:%s:container:flag", cid)
rep, err := gore.NewCommand("GET", flagKey).Run(conn)
if err != nil {
logs.Info("Status failed in get flag", err)
return
}
if !rep.IsNil() {
gore.NewCommand("DEL", flagKey).Run(conn)
logs.Debug(cid[:12], "Status flag set, ignore")
return
}
url := fmt.Sprintf("%s/api/container/%s/kill", g.Config.Eru.Endpoint, cid)
utils.DoPut(url)
logs.Debug(cid[:12], "dead, remove from watching list")
}
func (self *FalconClient) Send(data map[string]float64, endpoint, tag string, timestamp, step int64) error {
metrics := []*model.MetricValue{}
var metric *model.MetricValue
for k, v := range data {
metric = &model.MetricValue{
Endpoint: endpoint,
Metric: k,
Value: v,
Step: step,
Type: "GAUGE",
Tags: tag,
Timestamp: timestamp,
}
metrics = append(metrics, metric)
}
logs.Debug(metrics)
var resp model.TransferResponse
if err := self.call("Transfer.Update", metrics, &resp); err != nil {
return err
}
logs.Debug(endpoint, timestamp, &resp)
return nil
}
func load(configPath string) {
if _, err := os.Stat(configPath); err != nil {
logs.Assert(err, "config file invaild")
}
b, err := ioutil.ReadFile(configPath)
if err != nil {
logs.Assert(err, "Read config file failed")
}
if err := yaml.Unmarshal(b, &Config); err != nil {
logs.Assert(err, "Load config file failed")
}
logs.Debug("Configure:", Config)
}
func softOOMKill(cid string, rate float64) {
logs.Debug("OOM killed", cid[:12])
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
logs.Info("OOM killed set flag", err)
}
if err := g.Docker.StopContainer(cid, 10); err != nil {
logs.Info("OOM killed failed", cid[:12])
return
}
logs.Info("OOM killed success", cid[:12])
}
func (self *UpStream) createTCPConn() error {
self.scheme = "tcp"
tcpAddr, err := net.ResolveTCPAddr("tcp", self.addr)
if err != nil {
logs.Info("Resolve", self.addr, "failed", err)
return err
}
conn, err := net.DialTCP("tcp", nil, tcpAddr)
if err != nil {
logs.Debug("Connect backend failed", err)
return err
}
self.tcplog = conn
self.Close = self.tcplog.Close
return nil
}
func judgeMemoryUsage() {
var totalUsage uint64 = 0
var rate map[string]float64 = make(map[string]float64)
for cid, usage := range usage {
totalUsage = totalUsage + usage
//TODO ugly
if v, ok := Apps[cid].Extend["__memory__"]; !ok {
rate[cid] = 0.0
continue
} else {
define, _ := v.(float64)
rate[cid] = float64(usage) / define
}
}
logs.Debug("Current memory usage", totalUsage, "max", g.Config.Limit.Memory)
for {
if totalUsage < g.Config.Limit.Memory {
return
}
var exceedRate float64 = 0.0
var cid string = ""
for k, v := range rate {
if exceedRate >= v {
continue
}
exceedRate = v
cid = k
}
if cid == "" {
logs.Info("MemLimit can not stop containers")
break
}
softOOMKill(cid, exceedRate)
totalUsage -= usage[cid]
delete(rate, cid)
}
for k, _ := range usage {
delete(usage, k)
}
}
func (self *EruApp) InitMetric() bool {
var err error
if self.Exec, err = g.Docker.CreateExec(
docker.CreateExecOptions{
AttachStdout: true,
Cmd: []string{
"cat", "/proc/net/dev",
},
Container: self.ID,
},
); err != nil {
logs.Info("Create exec failed", err)
return false
}
logs.Debug("Create exec id", self.Exec.ID[:12])
if !self.updateStats() {
return false
}
self.Last = time.Now()
self.saveLast()
return true
}
func delVLan(link netlink.Link) {
if err := netlink.LinkDel(link); err != nil {
logs.Debug("Delete device failed", err)
}
}
func reportContainerCure(cid string) {
url := fmt.Sprintf("%s/api/container/%s/cure", g.Config.Eru.Endpoint, cid)
utils.DoPut(url)
logs.Debug(cid[:12], "cured, added in watching list")
}
func NewRequest(r *http.Request) *Request {
req := &Request{*r, 0, 20}
req.Init()
logs.Debug("HTTP request", req.Method, req.URL.Path)
return req
}
func Streamer(route *defines.Route, logstream chan *defines.Log) {
var upstreams map[string]*UpStream = map[string]*UpStream{}
var types map[string]struct{}
var count int64 = 0
if route.Source != nil {
types = make(map[string]struct{})
for _, t := range route.Source.Types {
types[t] = struct{}{}
}
}
defer func() {
logs.Debug("Flush", route.ID, "cache logs")
for _, remote := range upstreams {
remote.Flush()
for _, log := range remote.Tail() {
logs.Info("Streamer can't send to remote", log)
}
remote.Close()
}
route.Done <- struct{}{}
}()
for logline := range logstream {
if types != nil {
if _, ok := types[logline.Type]; !ok {
continue
}
}
logline.Tag = route.Target.AppendTag
logline.Count = count
if g.Config.Lenz.Stdout {
logs.Info("Debug Output", logline)
continue
}
var f bool = false
for offset := 0; offset < route.Backends.Len(); offset++ {
addr, err := route.Backends.Get(logline.Name, offset)
if err != nil {
logs.Info("Get backend failed", err, logline.Name, logline.Data)
break
}
if _, ok := upstreams[addr]; !ok {
if ups, err := NewUpStream(addr); err != nil || ups == nil {
route.Backends.Remove(addr)
continue
} else {
upstreams[addr] = ups
}
}
f = true
if err := upstreams[addr].WriteData(logline); err != nil {
logs.Info("Sent to remote failed", err)
upstreams[addr].Close()
go func(upstream *UpStream) {
for _, log := range upstream.Tail() {
logstream <- log
}
}(upstreams[addr])
delete(upstreams, addr)
continue
}
//logs.Debug("Lenz Send", logline.Name, logline.EntryPoint, logline.ID, "to", addr)
break
}
if !f {
logs.Info("Lenz failed", logline.ID[:12], logline.Name, logline.EntryPoint, logline.Data)
}
if count == math.MaxInt64 {
count = 0
} else {
count++
}
}
}