func readConf() *conf.Conf {
conf := &conf.Conf{
Freq: 15,
}
loc := *flagConf
if *flagConf == "" {
p, err := exePath()
if err != nil {
slog.Error(err)
return conf
}
dir := filepath.Dir(p)
loc = filepath.Join(dir, "scollector.toml")
}
f, err := os.Open(loc)
if err != nil {
if *flagConf != "" {
slog.Fatal(err)
}
if *flagDebug {
slog.Error(err)
}
} else {
defer f.Close()
md, err := toml.DecodeReader(f, conf)
if err != nil {
slog.Fatal(err)
}
if u := md.Undecoded(); len(u) > 0 {
slog.Fatalf("extra keys in %s: %v", loc, u)
}
}
return conf
}
func readConf() *Conf {
conf := &Conf{
Freq: 15,
}
loc := *flagConf
if *flagConf == "" {
p, err := exePath()
if err != nil {
slog.Error(err)
return conf
}
dir := filepath.Dir(p)
loc = filepath.Join(dir, "scollector.toml")
}
f, err := os.Open(loc)
if err != nil {
if *flagConf != "" {
slog.Fatal(err)
}
if *flagDebug {
slog.Error(err)
}
} else {
defer f.Close()
_, err := toml.DecodeReader(f, conf)
if err != nil {
slog.Fatal(err)
}
}
return conf
}
func (s *Schedule) ActionNotify(at models.ActionType, user, message string, aks []models.AlertKey) error {
groupings, err := s.groupActionNotifications(aks)
if err != nil {
return err
}
for notification, states := range groupings {
incidents := []*models.IncidentState{}
for _, state := range states {
incidents = append(incidents, state)
}
data := actionNotificationContext{incidents, user, message, at, s}
buf := &bytes.Buffer{}
err := actionNotificationSubjectTemplate.Execute(buf, data)
if err != nil {
slog.Error("Error rendering action notification subject", err)
}
subject := buf.String()
buf = &bytes.Buffer{}
err = actionNotificationBodyTemplate.Execute(buf, data)
if err != nil {
slog.Error("Error rendering action notification body", err)
}
notification.Notify(subject, buf.String(), []byte(subject), buf.Bytes(), s.SystemConf, "actionNotification")
}
return nil
}
func metaWindowsIfaces() {
var dstConfigs []Win32_NetworkAdapterConfiguration
q := wmi.CreateQuery(&dstConfigs, "WHERE MACAddress != null")
err := wmi.Query(q, &dstConfigs)
if err != nil {
slog.Error(err)
return
}
mNicConfigs := make(map[uint32]*Win32_NetworkAdapterConfiguration)
for i, nic := range dstConfigs {
mNicConfigs[nic.InterfaceIndex] = &dstConfigs[i]
}
mNicTeamIDtoSpeed := make(map[string]uint64)
mNicTeamIDtoMaster := make(map[string]string)
var dstTeamMembers []MSFT_NetLbfoTeamMember
q = wmi.CreateQuery(&dstTeamMembers, "")
err = wmi.QueryNamespace(q, &dstTeamMembers, "root\\StandardCimv2")
if err == nil {
for _, teamMember := range dstTeamMembers {
mNicTeamIDtoSpeed[teamMember.InstanceID] = teamMember.ReceiveLinkSpeed
mNicTeamIDtoMaster[teamMember.InstanceID] = teamMember.Team
}
}
var dstAdapters []Win32_NetworkAdapter
q = wmi.CreateQuery(&dstAdapters, "WHERE PhysicalAdapter=True and MACAddress <> null and NetConnectionStatus = 2") //Only adapters with MAC addresses and status="Connected"
err = wmi.Query(q, &dstAdapters)
if err != nil {
slog.Error(err)
return
}
for _, v := range dstAdapters {
tag := opentsdb.TagSet{"iface": fmt.Sprint("Interface", v.InterfaceIndex)}
AddMeta("", tag, "description", v.Description, true)
AddMeta("", tag, "name", v.NetConnectionID, true)
AddMeta("", tag, "mac", strings.Replace(v.MACAddress, ":", "", -1), true)
if v.Speed != nil && *v.Speed != 0 {
AddMeta("", tag, "speed", v.Speed, true)
} else {
nicSpeed := mNicTeamIDtoSpeed[v.GUID]
AddMeta("", tag, "speed", nicSpeed, true)
}
nicMaster := mNicTeamIDtoMaster[v.GUID]
if nicMaster != "" {
AddMeta("", tag, "master", nicMaster, true)
}
nicConfig := mNicConfigs[v.InterfaceIndex]
if nicConfig != nil {
for _, ip := range *nicConfig.IPAddress {
AddMeta("", tag, "addr", ip, true) // blocked by array support in WMI See https://github.com/StackExchange/wmi/issues/5
}
}
}
}
// CheckNotifications processes past notification events. It returns the next time a notification is needed.
func (s *Schedule) CheckNotifications() time.Time {
silenced := s.Silenced()
s.Lock("CheckNotifications")
defer s.Unlock()
latestTime := utcNow()
notifications, err := s.DataAccess.Notifications().GetDueNotifications()
if err != nil {
slog.Error("Error getting notifications", err)
return utcNow().Add(time.Minute)
}
for ak, ns := range notifications {
if si := silenced(ak); si != nil {
slog.Infoln("silencing", ak)
continue
}
for name, t := range ns {
n := s.RuleConf.GetNotification(name)
if n == nil {
continue
}
//If alert is currently unevaluated because of a dependency,
//simply requeue it until the dependency resolves itself.
_, uneval := s.GetUnknownAndUnevaluatedAlertKeys(ak.Name())
unevaluated := false
for _, un := range uneval {
if un == ak {
unevaluated = true
break
}
}
if unevaluated {
s.QueueNotification(ak, n, t.Add(time.Minute))
continue
}
st, err := s.DataAccess.State().GetLatestIncident(ak)
if err != nil {
slog.Error(err)
continue
}
if st == nil {
continue
}
s.Notify(st, n)
}
}
s.sendNotifications(silenced)
s.pendingNotifications = nil
err = s.DataAccess.Notifications().ClearNotificationsBefore(latestTime)
if err != nil {
slog.Error("Error clearing notifications", err)
return utcNow().Add(time.Minute)
}
timeout, err := s.DataAccess.Notifications().GetNextNotificationTime()
if err != nil {
slog.Error("Error getting next notification time", err)
return utcNow().Add(time.Minute)
}
return timeout
}
func (n *Notification) DoGet() {
resp, err := http.Get(n.Get.String())
if err != nil {
slog.Error(err)
return
}
if resp.StatusCode >= 300 {
slog.Error("bad response on notification get:", resp.Status)
}
}
func sendBatch(batch []*opentsdb.DataPoint) {
if Print {
for _, d := range batch {
j, err := d.MarshalJSON()
if err != nil {
slog.Error(err)
}
slog.Info(string(j))
}
recordSent(len(batch))
return
}
now := time.Now()
resp, err := SendDataPoints(batch, tsdbURLs[currentTsdbURL])
if err == nil {
defer resp.Body.Close()
}
d := time.Since(now).Nanoseconds() / 1e6
Sample("collect.post.duration", Tags, float64(d))
Add("collect.post.total_duration", Tags, d)
Add("collect.post.count", Tags, 1)
// Some problem with connecting to the server; retry later.
if err != nil || (resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK) {
if err != nil {
Add("collect.post.error", Tags, 1)
slog.Error(err)
// Switch endpoint if possible
currentTsdbURL = (currentTsdbURL + 1) % len(tsdbURLs)
} else if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
Add("collect.post.bad_status", Tags, 1)
slog.Errorln(resp.Status)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
slog.Error(err)
}
if len(body) > 0 {
slog.Error(string(body))
}
// Switch endpoint if possible
currentTsdbURL = (currentTsdbURL + 1) % len(tsdbURLs)
}
restored := 0
for _, msg := range batch {
restored++
tchan <- msg
}
d := time.Second * 5
Add("collect.post.restore", Tags, int64(restored))
slog.Infof("restored %d, sleeping %s", restored, d)
time.Sleep(d)
return
}
recordSent(len(batch))
}
func (n *Notification) DoGet(ak string) {
resp, err := http.Get(n.Get.String())
if err != nil {
slog.Error(err)
return
}
if resp.StatusCode >= 300 {
slog.Error("bad response on notification get:", resp.Status)
} else {
slog.Infof("get notification successful for alert %s. Response code %d.", ak, resp.StatusCode)
}
}
func c_snmp_cdp(community, host string) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
cdpEntries := make(map[string]*cdpCacheEntry)
deviceIdRaw, err := snmp_subtree(host, community, cdpCacheDeviceId)
if err != nil {
return md, err
}
for k, v := range deviceIdRaw {
ids := strings.Split(k, ".")
if len(ids) != 2 {
slog.Error("unexpected snmp cdpCacheEntry id")
continue
}
cdpEntries[ids[0]] = &cdpCacheEntry{}
cdpEntries[ids[0]].DeviceId = fmt.Sprintf("%s", v)
cdpEntries[ids[0]].InterfaceId = ids[1]
}
devicePortRaw, err := snmp_subtree(host, community, cdpCacheDevicePort)
for k, v := range devicePortRaw {
ids := strings.Split(k, ".")
if len(ids) != 2 {
slog.Error("unexpected snmp cdpCacheEntry id")
continue
}
if entry, ok := cdpEntries[ids[0]]; ok {
entry.DevicePort = fmt.Sprintf("%s", v)
}
}
byInterface := make(map[string][]*cdpCacheEntry)
for _, entry := range cdpEntries {
if _, ok := byInterface[entry.InterfaceId]; ok {
byInterface[entry.InterfaceId] = append(byInterface[entry.InterfaceId], entry)
} else {
byInterface[entry.InterfaceId] = []*cdpCacheEntry{entry}
}
}
for iface, entry := range byInterface {
j, err := json.Marshal(entry)
if err != nil {
return md, err
}
metadata.AddMeta("", opentsdb.TagSet{"host": host, "iface": iface}, "cdpCacheEntries", string(j), false)
}
if err != nil {
return md, nil
}
return md, nil
}
func IndexTSDB(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
slog.Error(err)
}
indexTSDB(r, body)
}
func c_omreport_ps() (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
readOmreport(func(fields []string) {
if len(fields) < 3 || fields[0] == "Index" {
return
}
id := strings.Replace(fields[0], ":", "_", -1)
ts := opentsdb.TagSet{"id": id}
Add(&md, "hw.ps", severity(fields[1]), ts, metadata.Gauge, metadata.Ok, descDellHWPS)
pm := &metadata.HWPowerSupply{}
if len(fields) < 6 {
return
}
if fields[4] != "" {
pm.RatedInputWattage = fields[4]
}
if fields[5] != "" {
pm.RatedOutputWattage = fields[5]
}
if j, err := json.Marshal(&pm); err == nil {
metadata.AddMeta("", ts, "psMeta", string(j), true)
} else {
slog.Error(err)
}
}, "chassis", "pwrsupplies")
return md, nil
}
func (c *Client) request(path string, s interface{}) error {
u := &url.URL{
Scheme: "https",
Host: c.baseAddr,
Path: path,
}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
slog.Error(err)
return err
}
req.Header.Set("Accept", "application/json")
resp, err := c.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
b, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("%v: %v: %v", req.URL, resp.Status, string(b))
}
d := json.NewDecoder(resp.Body)
if err := d.Decode(&s); err != nil {
return err
}
return nil
}
func queuer() {
for dp := range tchan {
qlock.Lock()
for {
if len(queue) > MaxQueueLen {
slock.Lock()
dropped++
slock.Unlock()
break
}
m, err := json.Marshal(dp)
if err != nil {
slog.Error(err)
} else {
queue = append(queue, m)
}
select {
case dp = <-tchan:
continue
default:
}
break
}
qlock.Unlock()
}
}
func sendMetadata(ms []Metasend) {
b, err := json.Marshal(&ms)
if err != nil {
slog.Error(err)
return
}
resp, err := http.Post(metahost, "application/json", bytes.NewBuffer(b))
if err != nil {
slog.Error(err)
return
}
if resp.StatusCode != 204 {
slog.Errorln("bad metadata return:", resp.Status)
return
}
}
func init() {
err := slog.SetSyslog("scollector")
if err != nil {
slog.Error(err)
}
slog.Infof("starting %s", version.GetVersionInfo("scollector"))
}
func (f *fastlyClient) request(path string, values url.Values, s interface{}) error {
u := &url.URL{
Scheme: "https",
Host: "api.fastly.com",
Path: path,
RawQuery: values.Encode(),
}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
slog.Error(err)
return err
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Fastly-Key", f.key)
resp, err := f.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
b, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("%v: %v: %v", req.URL, resp.Status, string(b))
}
d := json.NewDecoder(resp.Body)
if err := d.Decode(&s); err != nil {
return err
}
return nil
}
func JSON(h func(miniprofiler.Timer, http.ResponseWriter, *http.Request) (interface{}, error)) http.Handler {
return miniprofiler.NewHandler(func(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
d, err := h(t, w, r)
if err != nil {
serveError(w, err)
return
}
if d == nil {
return
}
buf := new(bytes.Buffer)
if err := json.NewEncoder(buf).Encode(d); err != nil {
slog.Error(err)
serveError(w, err)
return
}
var tw io.Writer = w
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
w.Header().Set("Content-Encoding", "gzip")
gz := gzip.NewWriter(w)
defer gz.Close()
tw = gz
}
if cb := r.FormValue("callback"); cb != "" {
w.Header().Add("Content-Type", "application/javascript")
tw.Write([]byte(cb + "("))
buf.WriteTo(tw)
tw.Write([]byte(")"))
return
}
w.Header().Add("Content-Type", "application/json")
buf.WriteTo(tw)
})
}
// Silenced returns all currently silenced AlertKeys and the time they will be
// unsilenced.
func (s *Schedule) Silenced() map[models.AlertKey]models.Silence {
aks := make(map[models.AlertKey]models.Silence)
now := time.Now()
silences, err := s.DataAccess.Silence().GetActiveSilences()
if err != nil {
slog.Error("Error fetching silences.", err)
return nil
}
for _, si := range silences {
if !si.ActiveAt(now) {
continue
}
s.Lock("Silence")
for ak := range s.status {
if si.Silenced(now, ak.Name(), ak.Group()) {
if aks[ak].End.Before(si.End) {
aks[ak] = *si
}
}
}
s.Unlock()
}
return aks
}
func (s *Search) Index(mdp opentsdb.MultiDataPoint) {
for _, dp := range mdp {
s.Lock()
metric := dp.Metric
key := metric + dp.Tags.String()
p := s.Last[key]
if p == nil {
p = &lastInfo{}
s.Last[key] = p
}
if p.timestamp < dp.Timestamp {
if fv, err := getFloat(dp.Value); err == nil {
p.diffFromPrev = (fv - p.lastVal) / float64(dp.Timestamp-p.timestamp)
p.lastVal = fv
} else {
slog.Error(err)
}
p.timestamp = dp.Timestamp
}
s.Unlock()
select {
case s.indexQueue <- dp:
default:
collect.Add("search.dropped", opentsdb.TagSet{}, 1)
}
}
}
func (s *Search) Index(mdp opentsdb.MultiDataPoint) {
for _, dp := range mdp {
s.Lock()
mmap := s.last[dp.Metric]
if mmap == nil {
mmap = make(map[string]*database.LastInfo)
s.last[dp.Metric] = mmap
}
p := mmap[dp.Tags.String()]
if p == nil {
p = &database.LastInfo{}
mmap[dp.Tags.String()] = p
}
if p.Timestamp < dp.Timestamp {
if fv, err := getFloat(dp.Value); err == nil {
p.DiffFromPrev = (fv - p.LastVal) / float64(dp.Timestamp-p.Timestamp)
p.LastVal = fv
} else {
slog.Error(err)
}
p.Timestamp = dp.Timestamp
}
s.Unlock()
select {
case s.indexQueue <- dp:
default:
collect.Add("search.dropped", opentsdb.TagSet{}, 1)
}
}
}
func (s *Schedule) PutMetadata(k metadata.Metakey, v interface{}) {
isCoreMeta := (k.Name == "desc" || k.Name == "unit" || k.Name == "rate")
if !isCoreMeta {
s.metaLock.Lock()
s.Metadata[k] = &Metavalue{time.Now().UTC(), v}
s.metaLock.Unlock()
return
}
if k.Metric == "" {
slog.Error("desc, rate, and unit require metric name")
return
}
strVal, ok := v.(string)
if !ok {
slog.Errorf("desc, rate, and unit require value to be string. Found: %s", reflect.TypeOf(v))
return
}
s.metricMetaLock.Lock()
metricData, ok := s.metricMetadata[k.Metric]
if !ok {
metricData = &MetadataMetric{}
s.metricMetadata[k.Metric] = metricData
}
switch k.Name {
case "desc":
metricData.Description = strVal
case "unit":
metricData.Unit = strVal
case "rate":
metricData.Type = strVal
}
s.metricMetaLock.Unlock()
}
// AddMeta adds a metadata entry to memory, which is queued for later sending.
func AddMeta(metric string, tags opentsdb.TagSet, name string, value interface{}, setHost bool) {
if tags == nil {
tags = make(opentsdb.TagSet)
}
if _, present := tags["host"]; setHost && !present {
tags["host"] = util.Hostname
}
if err := tags.Clean(); err != nil {
slog.Error(err)
return
}
ts := tags.Tags()
metalock.Lock()
defer metalock.Unlock()
prev, present := metadata[Metakey{metric, ts, name}]
if present && !reflect.DeepEqual(prev, value) {
slog.Infof("metadata changed for %s/%s/%s: %v to %v", metric, ts, name, prev, value)
go sendMetadata([]Metasend{{
Metric: metric,
Tags: tags,
Name: name,
Value: value,
}})
} else if metadebug {
slog.Infof("AddMeta for %s/%s/%s: %v", metric, ts, name, value)
}
metadata[Metakey{metric, ts, name}] = value
}
func (s *Schedule) getErrorCounts() (failing, total int) {
var err error
failing, total, err = s.DataAccess.Errors().GetFailingAlertCounts()
if err != nil {
slog.Error(err)
}
return
}
func (s *Schedule) markAlertError(name string, e error) {
d := s.DataAccess.Errors()
if err := d.MarkAlertFailure(name, e.Error()); err != nil {
slog.Error(err)
return
}
}
func (s *Schedule) AlertSuccessful(name string) bool {
b, err := s.DataAccess.Errors().IsAlertFailing(name)
if err != nil {
slog.Error(err)
b = true
}
return !b
}
func sendMetadata(ms []Metasend) {
b, err := json.Marshal(&ms)
if err != nil {
slog.Error(err)
return
}
resp, err := http.Post(metahosts[currentmetahost], "application/json", bytes.NewBuffer(b))
if err != nil {
slog.Error(err)
currentmetahost = (currentmetahost + 1) % len(metahosts)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
slog.Errorln("bad metadata return:", resp.Status)
return
}
}
func (s *Search) backupLoop() {
for {
time.Sleep(2 * time.Minute)
slog.Info("Backing up last data to redis")
err := s.BackupLast()
if err != nil {
slog.Error(err)
}
}
}
func c_meta_windows_version() (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
var dst []Win32_OperatingSystem
q := wmi.CreateQuery(&dst, "")
err := wmi.Query(q, &dst)
if err != nil {
slog.Error(err)
return md, err
}
var dstComputer []Win32_ComputerSystem
q = wmi.CreateQuery(&dstComputer, "")
err = wmi.Query(q, &dstComputer)
if err != nil {
slog.Error(err)
return md, err
}
var dstBIOS []Win32_BIOS
q = wmi.CreateQuery(&dstBIOS, "")
err = wmi.Query(q, &dstBIOS)
if err != nil {
slog.Error(err)
return md, err
}
for _, v := range dst {
metadata.AddMeta("", nil, "version", v.Version, true)
metadata.AddMeta("", nil, "versionCaption", v.Caption, true)
}
for _, v := range dstComputer {
metadata.AddMeta("", nil, "manufacturer", v.Manufacturer, true)
metadata.AddMeta("", nil, "model", v.Model, true)
metadata.AddMeta("", nil, "memoryTotal", v.TotalPhysicalMemory, true)
}
for _, v := range dstBIOS {
metadata.AddMeta("", nil, "serialNumber", v.SerialNumber, true)
}
return md, nil
}
func c_bacula_status(user, pass, dbase string) (opentsdb.MultiDataPoint, error) {
dsn := fmt.Sprintf("%s:%s@/%s", user, pass, dbase)
db, err := sql.Open("mysql", dsn)
if err != nil {
slog.Error("Failed to connect to database")
return nil, err
}
defer db.Close()
var md opentsdb.MultiDataPoint
var name string
var value int
var tagSet opentsdb.TagSet
var rate metadata.RateType
var unit metadata.Unit
tagSet = nil
rate = metadata.Gauge
unit = metadata.Item
description := "Successful backup jobs in the last week"
rows, err := db.Query("SELECT DISTINCT(Name) from Job")
if err != nil {
slog.Error("Query Error: " + err.Error())
return nil, err
}
for rows.Next() {
rows.Scan(&name)
r := db.QueryRow("SELECT count(JobId) as value from Job where RealEndTime>SUBTIME(now(), '7 0:0:0') and JobStatus='T' and Name=?", name)
r.Scan(&value)
slog.Infoln(name, value)
Add(&md, "bacula."+name+".last_week", value, tagSet, rate, unit, description)
}
return md, nil
}
// winNetworkInit maintains a mapping of InstanceName to InterfaceIndex
func winNetworkInit() {
update := func() {
var dstNetworkAdapter []Win32_NetworkAdapter
q := wmi.CreateQuery(&dstNetworkAdapter, "WHERE PhysicalAdapter=True and MACAddress <> null")
err := queryWmi(q, &dstNetworkAdapter)
if err != nil {
slog.Error(err)
return
}
for _, nic := range dstNetworkAdapter {
var iface = fmt.Sprint("Interface", nic.InterfaceIndex)
// Get PnPName using Win32_PnPEntity class
var pnpname = ""
var escapeddeviceid = strings.Replace(nic.PNPDeviceID, "\\", "\\\\", -1)
var filter = fmt.Sprintf("WHERE DeviceID='%s'", escapeddeviceid)
var dstPnPName []Win32_PnPEntity
q = wmi.CreateQuery(&dstPnPName, filter)
err = queryWmi(q, &dstPnPName)
if err != nil {
slog.Error(err)
return
}
for _, pnp := range dstPnPName { // Really should be a single item
pnpname = pnp.Name
}
if pnpname == "" {
slog.Errorf("%s cannot find Win32_PnPEntity %s", iface, filter)
continue
}
// Convert to instance name (see http://goo.gl/jfq6pq )
instanceName := winNetworkToInstanceName(pnpname)
mNicInstanceNameToInterfaceIndex[instanceName] = iface
}
}
update()
go func() {
for range time.Tick(time.Minute * 5) {
update()
}
}()
}