func (s *SshCommand) Exec(cmd string, arguments ...string) (io.Reader, io.Reader, error) {
for _, arg := range arguments {
cmd += " " + arg
}
logger.Yellow("ssh", "Executing command '%s' on %s:%d as %s", cmd, s.Ssh.Host, s.Ssh.Port, s.Username)
conn, err := pool.Get(s.Ssh)
if err != nil {
return nil, nil, err
}
defer pool.Done(s.Ssh)
session, err := conn.NewSession()
if err != nil {
return nil, nil, err
}
defer session.Close()
var stdoutBuf, stderrBuf bytes.Buffer
session.Stdout = &stdoutBuf
session.Stderr = &stderrBuf
err = session.Run(cmd)
if err != nil {
return &stdoutBuf, &stderrBuf, err
}
return &stdoutBuf, &stderrBuf, nil
}
func (s *SshCommand) Dial(network string, address string) (net.Conn, error) {
conn, err := pool.Get(s.Ssh)
if err != nil {
return nil, err
}
defer pool.Done(s.Ssh) // FIXME: This can easily leak ssh connections
logger.Yellow("ssh", "Dialing %s://%s via ssh://%s@%s:%d", network, address, s.Ssh.Username, s.Ssh.Host, s.Ssh.Port)
return conn.Dial(network, address)
}
func loop() {
ticker := time.Tick(time.Second)
for t := range ticker {
pool.lock.Lock()
for s, conn := range pool.pool {
if t.Sub(conn.lastUse) > time.Second*10 && conn.refCount == 0 && conn.client != nil {
conn.client.Close()
conn.client = nil
logger.Yellow("ssh", "Closing unused connection %s:%d", s.Host, s.Port)
}
}
pool.lock.Unlock()
}
}
func Loop(wg sync.WaitGroup) {
_, err := GetHost("000000000000000000000000")
if err != nil {
p, found := plugins.GetPlugin("localtransport")
if !found {
logger.Red("monitor", "localtransport plugin not found")
}
host := Host{
Id: bson.ObjectIdHex("000000000000000000000000"),
Name: "localhost",
TransportId: "localtransport",
Transport: p().(plugins.Transport),
}
hostCollection.Insert(host)
logger.Yellow("monitor", "Added localhost transport with id %s", host.Id.String())
}
ticker := time.Tick(time.Millisecond * 100)
inFlight := make(map[bson.ObjectId]bool)
inFlightLock := sync.RWMutex{}
for t := range ticker {
var monitors []Monitor
err := monitorCollection.Find(bson.M{}).All(&monitors)
if err != nil {
logger.Red("monitor", "Error getting monitors from Mongo: %s", err.Error())
continue
}
for _, mon := range monitors {
age := t.Sub(mon.LastCheck) // positive: past
wait := mon.NextCheck.Sub(t) // positive: future
inFlightLock.RLock()
_, found := inFlight[mon.Id]
inFlightLock.RUnlock()
if found {
// skipping monitors in flight
} else if age > mon.Interval*2 && wait < -mon.Interval {
checkIn := time.Duration(rand.Int63n(int64(mon.Interval)))
mon.NextCheck = t.Add(checkIn)
logger.Yellow("monitor", "%s %s: Delaying first check by %s", mon.Id.Hex(), mon.Agent.AgentId, checkIn)
err = UpdateMonitor(&mon)
if err != nil {
logger.Red("Error updating: %v", err.Error())
}
} else if wait < 0 {
inFlightLock.Lock()
inFlight[mon.Id] = true
inFlightLock.Unlock()
go func(mon Monitor) {
var host Host
hostCollection.FindId(mon.HostId).One(&host)
r := mon.Agent.Run(host.Transport)
if r.Status == plugins.Ok {
logger.Green("monitor", "%s %s: %s [%s]: %s", mon.Id.Hex(), mon.Agent.AgentId, r.Text, r.Duration, r.Measurements)
} else {
logger.Red("monitor", "%s %s: %s [%s]", mon.Id.Hex(), mon.Agent.AgentId, r.Text, r.Duration)
}
mon.LastResult = r
mon.LastCheck = t
mon.NextCheck = t.Add(mon.Interval)
err = UpdateMonitor(&mon)
if err != nil {
logger.Red("monitor", "Error updating: %s", err.Error())
}
inFlightLock.Lock()
delete(inFlight, mon.Id)
inFlightLock.Unlock()
}(mon)
}
}
}
wg.Done()
}