func (m *AttachManager) Attach(app *defines.Meta) {
// Not Thread Safe
if m.Attached(app.ID) {
return
}
outrd, outwr := io.Pipe()
errrd, errwr := io.Pipe()
go func() {
err := g.Docker.AttachToContainer(docker.AttachToContainerOptions{
Container: app.ID,
OutputStream: outwr,
ErrorStream: errwr,
Stdin: false,
Stdout: true,
Stderr: true,
Stream: true,
})
outwr.Close()
errwr.Close()
logs.Debug("Lenz Attach", app.ID[:12], "finished")
if err != nil {
logs.Debug("Lenz Attach", app.ID, "failure:", err)
}
m.send(&defines.AttachEvent{Type: "detach", App: app})
m.Lock()
defer m.Unlock()
delete(m.attached, app.ID)
}()
m.Lock()
m.attached[app.ID] = NewLogPump(outrd, errrd, app)
m.Unlock()
m.send(&defines.AttachEvent{Type: "attach", App: app})
logs.Debug("Lenz Attach", app.ID[:12], "success")
}
func monitor() {
for event := range events {
switch event.Status {
case common.STATUS_DIE:
logs.Debug("Status", event.Status, event.ID[:12], event.From)
app.Remove(event.ID)
reportContainerDeath(event.ID)
case common.STATUS_START:
logs.Debug("Status", event.Status, event.ID[:12], event.From)
// if not in watching list, just ignore it
if meta := getContainerMeta(event.ID); meta != nil && !app.Valid(event.ID) {
container, err := g.Docker.InspectContainer(event.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
break
}
eruApp := app.NewEruApp(container, meta)
if eruApp == nil {
logs.Info("Create EruApp failed")
break
}
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(event.ID)
}
}
}
}
func load() {
containers, err := g.Docker.ListContainers(docker.ListContainersOptions{All: true})
if err != nil {
logs.Assert(err, "List containers")
}
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
containersKey := fmt.Sprintf("eru:agent:%s:containers:meta", g.Config.HostName)
logs.Debug("Status get targets from", containersKey)
rep, err := gore.NewCommand("HGETALL", containersKey).Run(conn)
if err != nil {
logs.Assert(err, "Status get targets")
}
if rep.IsNil() {
return
}
targets, err := rep.Map()
if err != nil {
logs.Assert(err, "Status load targets")
}
logs.Debug("Status targets:", targets)
logs.Info("Status load container")
for _, container := range containers {
if _, ok := targets[container.ID]; !ok {
continue
}
status := getStatus(container.Status)
if status != common.STATUS_START {
reportContainerDeath(container.ID)
continue
}
var meta map[string]interface{}
if err := json.Unmarshal([]byte(targets[container.ID]), &meta); err != nil {
logs.Info("Status load failed", err)
continue
}
c, err := g.Docker.InspectContainer(container.ID)
if err != nil {
logs.Info("Status inspect docker failed", err)
continue
}
if eruApp := app.NewEruApp(c, meta); eruApp != nil {
lenz.Attacher.Attach(&eruApp.Meta)
app.Add(eruApp)
reportContainerCure(container.ID)
}
}
}
func getClient(conn ssh.ConnMetadata) (*ssh.Client, error) {
Lock.RLock()
defer Lock.RUnlock()
meta := MetaData[conn.RemoteAddr()]
logs.Debug("Connection accepted from", conn.RemoteAddr())
return meta.Client, nil
}
func NewEruApp(container *docker.Container, extend map[string]interface{}) *EruApp {
name, entrypoint, ident := utils.GetAppInfo(container.Name)
if name == "" {
logs.Info("Container name invaild", container.Name)
return nil
}
logs.Debug("Eru App", name, entrypoint, ident)
transfer, _ := g.Transfers.Get(container.ID, 0)
client := falcon.CreateFalconClient(
transfer,
time.Duration(g.Config.Metrics.Timeout)*time.Millisecond,
)
step := time.Duration(g.Config.Metrics.Step) * time.Second
extend["hostname"] = g.Config.HostName
extend["cid"] = container.ID[:12]
extend["ident"] = ident
tag := []string{}
for k, v := range extend {
tag = append(tag, fmt.Sprintf("%s=%v", k, v))
}
endpoint := fmt.Sprintf("%s-%s", name, entrypoint)
meta := defines.Meta{container.ID, container.State.Pid, name, entrypoint, ident, extend}
metric := metric.CreateMetric(step, client, strings.Join(tag, ","), endpoint)
eruApp := &EruApp{meta, metric}
return eruApp
}
func closeConn(conn ssh.ConnMetadata) error {
Lock.Lock()
defer Lock.Unlock()
defer delete(MetaData, conn.RemoteAddr())
logs.Debug("Clean sessions")
return nil
}
func NewLogPump(stdout, stderr io.Reader, app *defines.Meta) *LogPump {
obj := &LogPump{
app: app,
channels: make(map[chan *defines.Log]struct{}),
}
pump := func(typ string, source io.Reader) {
buf := bufio.NewReader(source)
for {
data, err := buf.ReadBytes('\n')
if err != nil {
if err != io.EOF {
logs.Debug("Lenz Pump:", app.ID, typ, err)
}
return
}
obj.send(&defines.Log{
Data: strings.TrimSuffix(string(data), "\n"),
ID: app.ID,
Name: app.Name,
EntryPoint: app.EntryPoint,
Ident: app.Ident,
Type: typ,
Datetime: time.Now().Format(common.DATETIME_FORMAT),
})
}
}
go pump("stdout", stdout)
go pump("stderr", stderr)
return obj
}
func InitLenz() {
Attacher = NewAttachManager()
Router = NewRouteManager(Attacher)
Routefs = RouteFileStore(g.Config.Lenz.Routes)
if len(g.Config.Lenz.Forwards) > 0 {
logs.Debug("Lenz Routing all to", g.Config.Lenz.Forwards)
target := defines.Target{Addrs: g.Config.Lenz.Forwards}
route := defines.Route{ID: common.LENZ_DEFAULT, Target: &target}
route.LoadBackends()
Router.Add(&route)
}
if _, err := os.Stat(g.Config.Lenz.Routes); err == nil {
logs.Debug("Loading and persisting routes in", g.Config.Lenz.Routes)
logs.Assert(Router.Load(Routefs), "persistor")
}
logs.Info("Lenz initiated")
}
func DoPut(url string) {
req, err := http.NewRequest("PUT", url, nil)
if err != nil {
logs.Debug("Gen request failed", err)
return
}
response, err := httpClient.Do(req)
if err != nil {
logs.Debug("Do request failed", err)
return
}
defer response.Body.Close()
data, err := ioutil.ReadAll(response.Body)
if err != nil {
logs.Debug("Read response failed", err)
return
}
logs.Debug("Response:", string(data))
}
func reportContainerDeath(cid string) {
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
flagKey := fmt.Sprintf("eru:agent:%s:container:flag", cid)
rep, err := gore.NewCommand("GET", flagKey).Run(conn)
if err != nil {
logs.Info("Status failed in get flag", err)
return
}
if !rep.IsNil() {
gore.NewCommand("DEL", flagKey).Run(conn)
logs.Debug(cid[:12], "Status flag set, ignore")
return
}
url := fmt.Sprintf("%s/api/container/%s/kill/", g.Config.Eru.Endpoint, cid)
utils.DoPut(url)
logs.Debug(cid[:12], "dead, remove from watching list")
}
func LoadKey(keyPath string) (ssh.Signer, error) {
bytes, err := ioutil.ReadFile(keyPath)
if err != nil {
return nil, err
}
logs.Debug(keyPath, GetFingerPrint(bytes))
key, err := ssh.ParsePrivateKey(bytes)
if err != nil {
return nil, err
}
return key, nil
}
func softOOMKill(cid string, rate float64) {
logs.Debug("OOM killed", cid[:12])
conn := g.GetRedisConn()
defer g.ReleaseRedisConn(conn)
key := fmt.Sprintf("eru:agent:%s:container:reason", cid)
if _, err := gore.NewCommand("SET", key, common.OOM_KILLED).Run(conn); err != nil {
logs.Info("OOM killed set flag", err)
}
if err := g.Docker.StopContainer(cid, 10); err != nil {
logs.Info("OOM killed failed", cid[:12])
return
}
logs.Info("OOM killed success", cid[:12])
}
func load(configPath string) {
if _, err := os.Stat(configPath); err != nil {
logs.Assert(err, "config file invaild")
}
b, err := ioutil.ReadFile(configPath)
if err != nil {
logs.Assert(err, "Read config file failed")
}
if err := yaml.Unmarshal(b, &Config); err != nil {
logs.Assert(err, "Load config file failed")
}
logs.Debug("Configure:", Config)
}
func (self *UpStream) createTCPConn() error {
self.scheme = "tcp"
tcpAddr, err := net.ResolveTCPAddr("tcp", self.addr)
if err != nil {
logs.Info("Resolve", self.addr, "failed", err)
return err
}
conn, err := net.DialTCP("tcp", nil, tcpAddr)
if err != nil {
logs.Debug("Connect backend failed", err)
return err
}
self.conn = conn
self.encoder = json.NewEncoder(conn)
self.Close = conn.Close
return nil
}
func judgeMemoryUsage() {
var totalUsage uint64 = 0
var rate map[string]float64 = make(map[string]float64)
for cid, usage := range usage {
totalUsage = totalUsage + usage
//TODO ugly
if v, ok := Apps[cid].Extend["__memory__"]; !ok {
rate[cid] = 0.0
continue
} else {
define, _ := v.(float64)
rate[cid] = float64(usage) / define
}
}
logs.Debug("Current memory usage", totalUsage, "max", g.Config.Limit.Memory)
for {
if totalUsage < g.Config.Limit.Memory {
return
}
var exceedRate float64 = 0.0
var cid string = ""
for k, v := range rate {
if exceedRate >= v {
continue
}
exceedRate = v
cid = k
}
if cid == "" {
logs.Info("MemLimit can not stop containers")
break
}
softOOMKill(cid, exceedRate)
totalUsage -= usage[cid]
delete(rate, cid)
}
for k, _ := range usage {
delete(usage, k)
}
}
func (self *SSHConn) serve() error {
serverConn, chans, reqs, err := ssh.NewServerConn(self.Conn, self.config)
if err != nil {
return err
}
defer serverConn.Close()
clientConn, err := getClient(serverConn)
if err != nil {
return err
}
defer clientConn.Close()
go ssh.DiscardRequests(reqs)
for newChannel := range chans {
remoteChannel, remoteRequest, err := clientConn.OpenChannel(newChannel.ChannelType(), newChannel.ExtraData())
if err != nil {
return err
}
localChannel, localRequest, err := newChannel.Accept()
if err != nil {
return err
}
// connect requests
go func() {
logs.Debug("Waiting for request")
r:
for {
var req *ssh.Request
var dst ssh.Channel
select {
case req = <-localRequest:
dst = remoteChannel
logs.Debug("from local to remote")
case req = <-remoteRequest:
dst = localChannel
logs.Debug("from remote to local")
}
if req == nil {
break
}
logs.Debug("Request", req.Type, req.WantReply)
b, err := dst.SendRequest(req.Type, req.WantReply, req.Payload)
if err != nil {
logs.Info(err)
}
if req.WantReply {
req.Reply(b, nil)
}
switch req.Type {
case "exit-status":
break r
}
}
}()
// connect channels
logs.Debug("Connecting channels")
go func() {
defer remoteChannel.Close()
io.Copy(remoteChannel, localChannel)
remoteChannel.SendRequest("exit-status", false, []byte{0, 0, 0, 0})
}()
go func() {
defer localChannel.Close()
io.Copy(localChannel, remoteChannel)
localChannel.SendRequest("exit-status", false, []byte{0, 0, 0, 0})
}()
}
closeConn(serverConn)
return nil
}
func reportContainerCure(cid string) {
url := fmt.Sprintf("%s/api/container/%s/cure/", g.Config.Eru.Endpoint, cid)
utils.DoPut(url)
logs.Debug(cid[:12], "cured, added in watching list")
}
func NewRequest(r *http.Request) *Request {
req := &Request{*r, 0, 20}
req.Init()
logs.Debug("HTTP request", req.Method, req.URL.Path)
return req
}
func DelVlan(link netlink.Link) {
if err := netlink.LinkDel(link); err != nil {
logs.Debug("Delete device failed", err)
}
}
func Streamer(route *defines.Route, logstream chan *defines.Log) {
var upstreams map[string]*UpStream = map[string]*UpStream{}
var types map[string]struct{}
var count int64 = 0
if route.Source != nil {
types = make(map[string]struct{})
for _, t := range route.Source.Types {
types[t] = struct{}{}
}
}
defer func() {
logs.Debug("Flush", route.ID, "cache logs")
for _, remote := range upstreams {
remote.Flush()
for _, log := range remote.Tail() {
logs.Info("Streamer can't send to remote", log)
}
remote.Close()
}
route.Done <- struct{}{}
}()
for logline := range logstream {
if types != nil {
if _, ok := types[logline.Type]; !ok {
continue
}
}
logline.Tag = route.Target.AppendTag
logline.Count = count
if g.Config.Lenz.Stdout {
logs.Info("Debug Output", logline)
continue
}
var f bool = false
for offset := 0; offset < route.Backends.Len(); offset++ {
addr, err := route.Backends.Get(logline.Name, offset)
if err != nil {
logs.Info("Get backend failed", err, logline.Name, logline.Data)
break
}
if _, ok := upstreams[addr]; !ok {
if ups, err := NewUpStream(addr); err != nil || ups == nil {
route.Backends.Remove(addr)
continue
} else {
upstreams[addr] = ups
}
}
f = true
if err := upstreams[addr].WriteData(logline); err != nil {
logs.Info("Sent to remote failed", err)
upstreams[addr].Close()
go func(upstream *UpStream) {
for _, log := range upstream.Tail() {
logstream <- log
}
}(upstreams[addr])
delete(upstreams, addr)
continue
}
//logs.Debug("Lenz Send", logline.Name, logline.EntryPoint, logline.ID, "to", addr)
break
}
if !f {
logs.Info("Lenz failed", logline.ID[:12], logline.Name, logline.EntryPoint, logline.Data)
}
if count == math.MaxInt64 {
count = 0
} else {
count++
}
}
}
func InitSSHConfig() *ssh.ServerConfig {
privKey, err := utils.LoadKey(g.Config.PrivKey)
if err != nil {
logs.Assert(err, "Failed to load priv key")
}
config := &ssh.ServerConfig{
AuthLogCallback: func(conn ssh.ConnMetadata, method string, err error) {
logs.Debug("Method type", method, "Error", err)
},
PublicKeyCallback: func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
username := conn.User()
clientAddr := conn.RemoteAddr()
keyHex := utils.GetFingerPrint(key.Marshal())
user, remote := utils.GetRealUserRemote(username)
if user == "" || remote == "" {
return nil, errors.New("Wrong info")
}
logs.Info("Login attempt", conn.RemoteAddr(), username, user, remote, keyHex)
if !utils.CheckKey(user, keyHex) {
return nil, errors.New("Wrong key")
}
meta := defines.Meta{
Username: username,
Pubkey: key,
}
clientConfig := &ssh.ClientConfig{
User: "******",
Auth: []ssh.AuthMethod{
ssh.PublicKeys(privKey),
},
}
client, err := ssh.Dial("tcp", remote, clientConfig)
if err != nil {
return nil, err
}
logs.Info("Login success", remote)
meta.Remote = remote
meta.Client = client
Lock.Lock()
defer Lock.Unlock()
MetaData[clientAddr] = meta
return nil, nil
},
PasswordCallback: func(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
username := conn.User()
clientAddr := conn.RemoteAddr()
user, remote := utils.GetRealUserRemote(username)
if user == "" || remote == "" {
return nil, errors.New("Wrong info")
}
logs.Info("Login attempt", conn.RemoteAddr(), username, string(password))
meta := defines.Meta{
Username: username,
Password: string(password),
}
clientConfig := &ssh.ClientConfig{
User: "******",
Auth: []ssh.AuthMethod{
ssh.Password(string(password)),
},
}
client, err := ssh.Dial("tcp", remote, clientConfig)
if err != nil {
return nil, err
}
logs.Info("Login success", remote)
meta.Remote = remote
meta.Client = client
Lock.Lock()
defer Lock.Unlock()
MetaData[clientAddr] = meta
return nil, nil
},
}
hostKey, err := utils.LoadKey(g.Config.HostKey)
if err != nil {
logs.Assert(err, "Failed to load host key")
}
config.AddHostKey(hostKey)
return config
}
