/*
Loop for monitoring a probe
*/
func MonitorProbe(p Probe) {
var resp *http.Response // Http response
var req *http.Request // Http response
var body []byte // Http body
var err error // Error handling
var containers map[string]*dguard.Container // Returned container list
var lastContainers map[string]*dguard.Container // Old returned container list (used to compare running state)
var dbContainers []dguard.Container // Containers in DB
var tmpProbeInfos dguard.ProbeInfos // Temporary probe infos
// Reloading loop
for {
var statsToInsert []Stat // Stats to insert
lastContainers = containers
containers = nil
l.Verbose("Reloading", p.Name)
/*
GET PROBE INFOS
*/
// Make HTTP GET request
reqURI := p.URI + "/probeinfos"
l.Debug("MonitorProbe: GET", reqURI)
req, err = http.NewRequest("GET", reqURI, bytes.NewBufferString(""))
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): Can't create", p.Name, "HTTP request:", err)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
req.Header.Set("Auth", p.APIPassword)
// Do request
l.Debug("MonitorProbe: Get probe infos")
resp, err = HTTPClient.Do(req)
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): Can't get", p.Name, "probe infos:", err)
p.Infos.Running = false
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
if resp.StatusCode != 200 {
l.Error("MonitorProbe ("+p.Name+"): Probe returned a non 200 HTTP status code:", resp.StatusCode)
p.Infos.Running = false
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
// Get request body
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): Can't get", p.Name, "probe infos body:", err)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
l.Silly("MonitorProbe ("+p.Name+"):", "GET", reqURI, "body:\n", string(body))
// Parse body
err = json.Unmarshal([]byte(body), &(tmpProbeInfos))
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): Parsing probe infos:", err)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
tmpProbeInfos.Running = true
tmpProbeInfos.Name = p.Name
*(p.Infos) = tmpProbeInfos // Swap probe infos
/*
GET LIST OF CONTAINERS
*/
// Make HTTP GET request
reqURI = p.URI + "/list"
l.Debug("MonitorProbe: GET", reqURI)
req, err = http.NewRequest("GET", reqURI, bytes.NewBufferString(""))
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): Can't create", p.Name, "HTTP request:", err)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
req.Header.Set("Auth", p.APIPassword)
// Do request
l.Debug("MonitorProbe: Get list of containers")
resp, err = HTTPClient.Do(req)
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): Can't get", p.Name, "container list:", err)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
if resp.StatusCode != 200 {
l.Error("MonitorProbe ("+p.Name+"): Probe returned a non 200 HTTP status code:", resp.StatusCode)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
// Get request body
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): Can't get", p.Name, "container list body:", err)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
l.Silly("MonitorProbe ("+p.Name+"):", "GET", reqURI, "body:\n", string(body))
// Parse body
err = json.Unmarshal([]byte(body), &containers)
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): Parsing container list:", err)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
// Remove in DB old removed containers
l.Debug("MonitorProbe: GetContainersByProbe(", p.Name, ")")
dbContainers, err = GetContainersByProbe(p.Name)
if err != nil {
if err.Error() != "Not found" {
l.Error("MonitorProbe ("+p.Name+"): containers not found:", err)
time.Sleep(time.Second * time.Duration(p.ReloadTime))
continue
}
}
for _, dbC := range dbContainers {
var containerStillExist = false
dbC.Probe = p.Name
for _, c := range containers {
c.Probe = p.Name
if dbC.ID == c.ID {
containerStillExist = true
// Check if container started or stopped
c1, ok1 := containers[dbC.ID]
c2, ok2 := lastContainers[dbC.ID]
if ok1 && ok2 && (c1.Running != c2.Running) {
var event dguard.Event
var eventSeverity int
var eventType int
if c1.Running {
eventSeverity = dguard.EventNotice
eventType = dguard.EventContainerStarted
} else {
eventSeverity = dguard.EventCritical
eventType = dguard.EventContainerStopped
}
event = dguard.Event{
Severity: eventSeverity,
Type: eventType,
Target: dbC.Hostname + " (" + dbC.ID + ")",
Probe: p.Name,
Data: ""}
Alert(event)
}
}
}
if !containerStillExist {
var event = dguard.Event{
Severity: dguard.EventNotice,
Type: dguard.EventContainerRemoved,
Target: dbC.Hostname + " (" + dbC.ID + ")",
Probe: p.Name,
Data: ""}
DeleteContainer(&dbC)
Alert(event)
}
}
// Add containers and stats in DB
for _, c := range containers {
var newContainer = c
var id string
var tmpContainer dguard.Container
var newStat Stat
// Add containers in DB
c.Probe = p.Name
tmpContainer, err = GetContainerByCID(c.ID)
if err != nil {
if err.Error() == "Not found" {
var event dguard.Event
event = dguard.Event{
Severity: dguard.EventNotice,
Type: dguard.EventContainerCreated,
Target: newContainer.Hostname + " (" + newContainer.ID + ")",
Probe: p.Name,
Data: "Image: " + newContainer.Image}
Alert(event)
id = newContainer.ID
} else {
l.Error("MonitorProbe ("+p.Name+"): GetContainerById:", err)
continue
}
} else {
id = tmpContainer.ID
}
err = InsertContainer(newContainer)
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): container insert:", err)
continue
}
newStat = Stat{id,
time.Unix(int64(c.Time), 0),
float64(c.SizeRootFs),
float64(c.SizeRw),
float64(c.MemoryUsed),
float64(c.NetBandwithRX),
float64(c.NetBandwithTX),
float64(c.CPUUsage),
c.Running}
statsToInsert = append(statsToInsert, newStat)
}
err = InsertStats(statsToInsert, p.Name)
if err != nil {
l.Error("MonitorProbe ("+p.Name+"): insert stats:", err)
continue
}
// Update ProbeLastStats
var tmpLastStats []dguard.Container
for _, c := range containers {
tmpLastStats = append(tmpLastStats, *c)
}
ProbeLastStats[p.Name] = tmpLastStats
// Pause
time.Sleep(time.Second * time.Duration(p.ReloadTime))
}
}
/*
Handle GET /probeinfos
*/
func HTTPHandlerProbeinfos(w http.ResponseWriter, r *http.Request) {
var returnStr string // Returned string
var probeInfos dguard.ProbeInfos // DGC Probe infos
var stat syscall.Statfs_t // Syscall to get disk usage
var out []byte // Command output
var err error // Error handling
// Get load average
out, err = exec.Command("sh", "-c", "uptime | awk -F\"average:\" '{print $2}' | awk '{printf \"%s%s%s\",$1,$2,$3}'").Output()
if err != nil {
l.Error("HTTPHandlerProbeinfos: get load avg:", err)
http.Error(w, http.StatusText(500), 500)
return
}
probeInfos.LoadAvg = string(out)
// Get memory usage
out, err = exec.Command("sh", "-c", "cat /proc/meminfo | grep MemTotal | awk '{printf \"%d\",$2}'").Output()
if err != nil {
l.Error("HTTPHandlerProbeinfos: get mem total:", err)
http.Error(w, http.StatusText(500), 500)
return
}
probeInfos.MemoryTotal, err = utils.S2F(string(out))
if err != nil {
l.Error("HTTPHandlerProbeinfos: get mem total S2I:", err)
http.Error(w, http.StatusText(500), 500)
return
}
out, err = exec.Command("sh", "-c", "cat /proc/meminfo | grep MemAvailable | awk '{printf \"%d\",$2}'").Output()
if err != nil {
l.Error("HTTPHandlerProbeinfos: get mem available:", err)
http.Error(w, http.StatusText(500), 500)
return
}
probeInfos.MemoryAvailable, err = utils.S2F(string(out))
if err != nil {
l.Error("HTTPHandlerProbeinfos: get mem available S2I:", err)
http.Error(w, http.StatusText(500), 500)
return
}
probeInfos.MemoryTotal = probeInfos.MemoryTotal * 1024
probeInfos.MemoryAvailable = probeInfos.MemoryAvailable * 1024
// Get disk usage
syscall.Statfs("/", &stat)
probeInfos.DiskTotal = float64(stat.Blocks * uint64(stat.Bsize))
probeInfos.DiskAvailable = float64(stat.Bavail * uint64(stat.Bsize))
// probeInfos => json
tmpJSON, err := json.Marshal(probeInfos)
if err != nil {
l.Error("HTTPHandlerProbeinfos: marshal JSON:", err)
http.Error(w, http.StatusText(500), 500)
}
// Add json to the returned string
returnStr = string(tmpJSON)
fmt.Fprint(w, returnStr)
}