// parseTcollectorValue parses a tcollector-style line into a data point.
func parseTcollectorValue(line string) (*opentsdb.DataPoint, error) {
sp := strings.Fields(line)
if len(sp) < 3 {
return nil, fmt.Errorf("bad line: %s", line)
}
ts, err := strconv.ParseInt(sp[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("bad timestamp: %s", sp[1])
}
val, err := strconv.ParseFloat(sp[2], 64)
if err != nil {
return nil, fmt.Errorf("bad value: %s", sp[2])
}
if !opentsdb.ValidTag(sp[0]) {
return nil, fmt.Errorf("bad metric: %s", sp[0])
}
dp := opentsdb.DataPoint{
Metric: sp[0],
Timestamp: ts,
Value: val,
}
tags := opentsdb.TagSet{}
for _, tag := range sp[3:] {
ts, err := opentsdb.ParseTags(tag)
if err != nil {
return nil, fmt.Errorf("bad tag, metric %s: %v: %v", sp[0], tag, err)
}
tags.Merge(ts)
}
setExternalTags(tags)
dp.Tags = tags
return &dp, nil
}
func BenchmarkSimpleRewrite(b *testing.B) {
rule := &DenormalizationRule{
Metric: "a.b.c",
TagNames: []string{"host"},
}
tags := opentsdb.TagSet{"host": "foo-bar", "baz": "qwerty"}
dp := &opentsdb.DataPoint{
Metric: "a.b.c",
Timestamp: 42,
Value: 3,
Tags: tags.Copy(),
}
for i := 0; i < b.N; i++ {
err := rule.Translate(dp)
if err != nil {
b.Fatal(err)
}
//expectedName := "__foo-bar.a.b.c"
/* if dp.Metric != expectedName {
b.Errorf("metric name %s is not `%s`", dp.Metric, expectedName)
}
if dp.Timestamp != 42 {
b.Errorf("new metric timestamp does not match. %d != 42", dp.Timestamp)
}
if dp.Value != 3 {
b.Errorf("new metric value does not match. %d != 3", dp.Value)
}
if !dp.Tags.Equal(tags) {
b.Errorf("new metric tags do not match. %v != %v", dp.Tags, tags)
} */
}
}
func (to *TagOverride) AddTags(t opentsdb.TagSet) {
if to.tags == nil {
to.tags = t.Copy()
} else {
to.tags = to.tags.Merge(t)
}
}
// AddMeta adds a metadata entry to memory, which is queued for later sending.
func AddMeta(metric string, tags opentsdb.TagSet, name string, value interface{}, setHost bool) {
if tags == nil {
tags = make(opentsdb.TagSet)
}
if _, present := tags["host"]; setHost && !present {
tags["host"] = util.Hostname
}
if err := tags.Clean(); err != nil {
slog.Error(err)
return
}
ts := tags.Tags()
metalock.Lock()
defer metalock.Unlock()
prev, present := metadata[Metakey{metric, ts, name}]
if present && !reflect.DeepEqual(prev, value) {
slog.Infof("metadata changed for %s/%s/%s: %v to %v", metric, ts, name, prev, value)
go sendMetadata([]Metasend{{
Metric: metric,
Tags: tags,
Name: name,
Value: value,
}})
} else if metadebug {
slog.Infof("AddMeta for %s/%s/%s: %v", metric, ts, name, value)
}
metadata[Metakey{metric, ts, name}] = value
}
// AddTS is the same as Add but lets you specify the timestamp
func AddTS(md *opentsdb.MultiDataPoint, name string, ts int64, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
if b, ok := value.(bool); ok {
if b {
value = 1
} else {
value = 0
}
}
tags := t.Copy()
if host, present := tags["host"]; !present {
tags["host"] = util.Hostname
} else if host == "" {
delete(tags, "host")
}
if rate != metadata.Unknown {
metadata.AddMeta(name, nil, "rate", rate, false)
}
if unit != metadata.None {
metadata.AddMeta(name, nil, "unit", unit, false)
}
if desc != "" {
metadata.AddMeta(name, tags, "desc", desc, false)
}
tags = AddTags.Copy().Merge(tags)
d := opentsdb.DataPoint{
Metric: name,
Timestamp: ts,
Value: value,
Tags: tags,
}
*md = append(*md, &d)
}
// Put is useful for capturing "events" that have a gauge value. Subsequent
// calls between the sending interval will overwrite previous calls.
func Put(metric string, ts opentsdb.TagSet, v interface{}) error {
if err := check(metric, &ts); err != nil {
return err
}
tss := metric + ts.String()
mlock.Lock()
puts[tss] = &putMetric{metric, ts.Copy(), v}
mlock.Unlock()
return nil
}
// Set registers a callback for the given metric and tags, calling f immediately
// before queueing data for send.
func Set(metric string, ts opentsdb.TagSet, f func() interface{}) error {
if err := check(metric, &ts); err != nil {
return err
}
tss := metric + ts.String()
mlock.Lock()
sets[tss] = &setMetric{metric, ts.Copy(), f}
mlock.Unlock()
return nil
}
// AddTS is the same as Add but lets you specify the timestamp
func AddTS(md *opentsdb.MultiDataPoint, name string, ts int64, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
// Check if we really want that metric
if skipMetric(name) {
return
}
tags := t.Copy()
if host, present := tags["host"]; !present {
tags["host"] = util.Hostname
} else if host == "" {
delete(tags, "host")
}
// if tags are not cleanable, log a message and skip it
if err := tags.Clean(); err != nil {
line := ""
//attempt to log where Add was called from
if _, filename, l, ok := runtime.Caller(1); ok {
if filepath.Base(filename) == "collectors.go" {
_, filename, l, ok = runtime.Caller(2)
}
if ok {
line = fmt.Sprintf("%s:%d", filepath.Base(filename), l)
}
}
slog.Errorf("Invalid tagset discovered: %s. Skipping datapoint. Added from: %s", tags.String(), line)
return
}
if rate != metadata.Unknown {
metadata.AddMeta(name, nil, "rate", rate, false)
}
if unit != metadata.None {
metadata.AddMeta(name, nil, "unit", unit, false)
}
if desc != "" {
metadata.AddMeta(name, tags, "desc", desc, false)
}
tags = AddTags.Copy().Merge(tags)
if b, ok := value.(bool); ok {
if b {
value = 1
} else {
value = 0
}
}
d := opentsdb.DataPoint{
Metric: name,
Timestamp: ts,
Value: value,
Tags: tags,
}
*md = append(*md, &d)
}
// GetLast returns the value of the most recent data point for the given metric
// and tag. tags should be of the form "{key=val,key2=val2}". If diff is true,
// the value is treated as a counter. err is non nil if there is no match.
func (s *Search) GetLast(metric string, tags opentsdb.TagSet, diff bool) (v float64, err error) {
s.RLock()
defer s.RUnlock()
if mmap := s.last[metric]; mmap != nil {
if p := mmap[tags.String()]; p != nil {
if diff {
return p.DiffFromPrev, nil
}
return p.LastVal, nil
}
}
return 0, nil
}
func Sample(metric string, ts opentsdb.TagSet, v float64) error {
if err := check(metric, &ts); err != nil {
return err
}
tss := metric + ts.String()
mlock.Lock()
if aggs[tss] == nil {
aggs[tss] = &agMetric{
metric: metric,
ts: ts.Copy(),
}
}
aggs[tss].values = append(aggs[tss].values, v)
mlock.Unlock()
return nil
}
// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc int64) error {
if err := check(metric, &ts); err != nil {
return err
}
tss := metric + ts.String()
mlock.Lock()
if counters[tss] == nil {
counters[tss] = &addMetric{
metric: metric,
ts: ts.Copy(),
}
}
counters[tss].value += inc
mlock.Unlock()
return nil
}
func c_fortinet_os(host, community string, cpuIntegrators map[string]tsIntegrator) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
ts := opentsdb.TagSet{"host": host}
// CPU
cpuRaw, err := snmp_subtree(host, community, fortinetBaseOID+fortinetCPU)
if err != nil {
return md, err
}
coreCount := len(cpuRaw)
var totalPercent int
for id, v := range cpuRaw {
cpuVal, err := strconv.Atoi(fmt.Sprintf("%v", v))
if err != nil {
return md, fmt.Errorf("couldn't convert cpu value to int for fortinet cpu utilization on host %v: %v", host, err)
}
ts := ts.Copy().Merge(opentsdb.TagSet{"processor": id})
Add(&md, "fortinet.cpu.percent_used", cpuVal, ts, metadata.Gauge, metadata.Pct, "")
totalPercent += cpuVal
}
if _, ok := cpuIntegrators[host]; !ok {
cpuIntegrators[host] = getTsIntegrator()
}
Add(&md, osCPU, cpuIntegrators[host](time.Now().Unix(), float64(totalPercent)/float64(coreCount)), opentsdb.TagSet{"host": host}, metadata.Counter, metadata.Pct, "")
// Memory
memTotal, err := snmp_oid(host, community, fortinetBaseOID+fortinetMemTotal)
if err != nil {
return md, fmt.Errorf("failed to get total memory for fortinet host %v: %v", host, err)
}
memTotalBytes := memTotal.Int64() * 2 << 9 // KiB to Bytes
Add(&md, "fortinet.mem.total", memTotal, ts, metadata.Gauge, metadata.KBytes, "The total memory in kilobytes.")
Add(&md, osMemTotal, memTotalBytes, ts, metadata.Gauge, metadata.Bytes, osMemTotalDesc)
memPctUsed, err := snmp_oid(host, community, fortinetBaseOID+fortinetMemPercentUsed)
if err != nil {
return md, fmt.Errorf("failed to get percent of memory used for fortinet host %v: %v", host, err)
}
Add(&md, "fortinet.mem.percent_used", memPctUsed, ts, metadata.Gauge, metadata.Pct, "The percent of memory used.")
memPctUsedFloat := float64(memPctUsed.Int64()) / 100
memPctFree := 100 - memPctUsed.Int64()
Add(&md, osMemPctFree, memPctFree, ts, metadata.Gauge, metadata.Pct, osMemPctFreeDesc)
memFree := float64(memTotalBytes) * (float64(1) - memPctUsedFloat)
Add(&md, osMemFree, int64(memFree), ts, metadata.Gauge, metadata.Bytes, osMemFreeDesc)
Add(&md, osMemUsed, int64(float64(memTotalBytes)-memFree), ts, metadata.Gauge, metadata.Bytes, osMemUsedDesc)
return md, nil
}
func (to *TagOverride) AddTagOverrides(sources map[string]string, t opentsdb.TagSet) error {
if to.matchedTags == nil {
to.matchedTags = make(map[string]*regexp.Regexp)
}
var err error
for tag, re := range sources {
to.matchedTags[tag], err = regexp.Compile(re)
if err != nil {
return fmt.Errorf("invalid regexp: %s error: %s", re, err)
}
}
if to.tags == nil {
to.tags = t.Copy()
} else {
to.tags = to.tags.Merge(t)
}
return nil
}
func TestTagMetadata_RoundTrip(t *testing.T) {
host := randString(4)
tagset := opentsdb.TagSet{"host": host, "iface": "foo", "iname": "bar", "direction": "in"}
if err := testData.Metadata().PutTagMetadata(tagset, "alias", "foo", time.Now()); err != nil {
t.Fatal(err)
}
metas, err := testData.Metadata().GetTagMetadata(tagset, "alias")
if err != nil {
t.Fatal(err)
}
if len(metas) != 1 {
t.Fatal("expected 1 metadata result")
}
m := metas[0]
if m.Name != "alias" {
t.Fatalf("name %s != alias", m.Name)
}
if !m.Tags.Equal(tagset) {
t.Fatalf("tagset %s != %s", m.Tags.String(), tagset.String())
}
}
func rabbitmqBackingQueueStatus(p string, ts opentsdb.TagSet, bqs rmqBackingQueueStatus) opentsdb.MultiDataPoint {
var md opentsdb.MultiDataPoint
Add(&md, p+"avg_rate", bqs.AvgAckEgressRate, ts.Copy().Merge(opentsdb.TagSet{"method": "ack", "direction": "out"}),
metadata.Rate, metadata.Message, DescRmqBackingQueueStatusAvgAckEgressRate)
Add(&md, p+"avg_rate", bqs.AvgAckIngressRate, ts.Copy().Merge(opentsdb.TagSet{"method": "ack", "direction": "in"}),
metadata.Rate, metadata.Message, DescRmqBackingQueueStatusAvgAckIngressRate)
Add(&md, p+"avg_rate", bqs.AvgEgressRate, ts.Copy().Merge(opentsdb.TagSet{"method": "noack", "direction": "out"}),
metadata.Rate, metadata.Message, DescRmqBackingQueueStatusAvgEgressRate)
Add(&md, p+"avg_rate", bqs.AvgIngressRate, ts.Copy().Merge(opentsdb.TagSet{"method": "noack", "direction": "in"}),
metadata.Rate, metadata.Message, DescRmqBackingQueueStatusAvgIngressRate)
Add(&md, p+"len", bqs.Len, ts,
metadata.Gauge, metadata.Message, DescRmqBackingQueueStatusLen)
return md
}
func haproxyFetch(user, pwd, tier, url string) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
var err error
const metric = "haproxy"
parse := func(v string) (int64, error) {
var i int64
if v != "" {
i, err = strconv.ParseInt(v, 10, 64)
if err != nil {
return 0, err
}
return i, nil
}
return i, nil
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
// Close connection after request. Default cached connections will get
// failures in the event of server closing idle connections.
// See https://github.com/golang/go/issues/8946
req.Close = true
req.SetBasicAuth(user, pwd)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
reader := csv.NewReader(resp.Body)
records, err := reader.ReadAll()
if err != nil {
return nil, err
}
if len(records) < 2 {
return nil, nil
}
for _, rec := range records[1:] {
// There is a trailing comma in haproxy's csv
if len(rec) != len(haproxyCSVMeta)+1 {
return nil, fmt.Errorf("expected %v lines. got: %v",
len(haproxyCSVMeta)+1, len(rec))
}
hType := haproxyType[rec[32]]
pxname := rec[0]
svname := rec[1]
ts := opentsdb.TagSet{"pxname": pxname, "svname": svname, "tier": tier}
for i, field := range haproxyCSVMeta {
m := strings.Join([]string{metric, hType, field.Name}, ".")
value := rec[i]
if field.Ignore == true {
continue
} else if strings.HasPrefix(field.Name, "hrsp") {
sp := strings.Split(field.Name, "_")
if len(sp) != 2 {
return nil, fmt.Errorf("unexpected field name %v in hrsp", field.Name)
}
ts := ts.Copy().Merge(opentsdb.TagSet{"status_code": sp[1]})
m = strings.Join([]string{metric, hType, sp[0]}, ".")
v, err := parse(value)
if err != nil {
return nil, err
}
Add(&md, m, v, ts, metadata.Counter, metadata.Response,
fmt.Sprintf("The number of http responses with a %v status code.", sp[1]))
} else if field.Name == "status" {
v, ok := haproxyStatus[value]
// Not distinging between MAINT and MAINT via...
if !ok {
v = 3
}
Add(&md, m, v, ts, field.RateType, field.Unit, field.Desc)
} else if field.Name == "check_status" {
if value == "" {
continue
}
v, ok := haproxyCheckStatus[value]
if !ok {
return nil, fmt.Errorf("unknown check status %v", value)
}
Add(&md, m, v, ts, field.RateType, field.Unit, field.Desc)
} else {
v, err := parse(value)
if err != nil {
return nil, err
}
Add(&md, m, v, ts, field.RateType, field.Unit, field.Desc)
}
}
}
return md, nil
}
// CollectStates sends various state information to bosun with collect.
func (s *Schedule) CollectStates() {
// [AlertName][Severity]Count
severityCounts := make(map[string]map[string]int64)
abnormalCounts := make(map[string]map[string]int64)
ackStatusCounts := make(map[string]map[bool]int64)
ackByNotificationCounts := make(map[string]map[bool]int64)
unAckOldestByNotification := make(map[string]time.Time)
activeStatusCounts := make(map[string]map[bool]int64)
// Initalize the Counts
for _, alert := range s.Conf.Alerts {
severityCounts[alert.Name] = make(map[string]int64)
abnormalCounts[alert.Name] = make(map[string]int64)
var i models.Status
for i = 1; i.String() != "none"; i++ {
severityCounts[alert.Name][i.String()] = 0
abnormalCounts[alert.Name][i.String()] = 0
}
ackStatusCounts[alert.Name] = make(map[bool]int64)
activeStatusCounts[alert.Name] = make(map[bool]int64)
ackStatusCounts[alert.Name][false] = 0
activeStatusCounts[alert.Name][false] = 0
ackStatusCounts[alert.Name][true] = 0
activeStatusCounts[alert.Name][true] = 0
}
for notificationName := range s.Conf.Notifications {
unAckOldestByNotification[notificationName] = time.Unix(1<<63-62135596801, 999999999)
ackByNotificationCounts[notificationName] = make(map[bool]int64)
ackByNotificationCounts[notificationName][false] = 0
ackByNotificationCounts[notificationName][true] = 0
}
//TODO:
// for _, state := range s.status {
// if !state.Open {
// continue
// }
// name := state.AlertKey.Name()
// alertDef := s.Conf.Alerts[name]
// nots := make(map[string]bool)
// for name := range alertDef.WarnNotification.Get(s.Conf, state.Group) {
// nots[name] = true
// }
// for name := range alertDef.CritNotification.Get(s.Conf, state.Group) {
// nots[name] = true
// }
// incident, err := s.GetIncident(state.Last().IncidentId)
// if err != nil {
// slog.Errorln(err)
// }
// for notificationName := range nots {
// ackByNotificationCounts[notificationName][state.NeedAck]++
// if incident != nil && incident.Start.Before(unAckOldestByNotification[notificationName]) && state.NeedAck {
// unAckOldestByNotification[notificationName] = incident.Start
// }
// }
// severity := state.CurrentStatus.String()
// lastAbnormal := state.LastAbnormalStatus.String()
// severityCounts[state.Alert][severity]++
// abnormalCounts[state.Alert][lastAbnormal]++
// ackStatusCounts[state.Alert][state.NeedAck]++
// activeStatusCounts[state.Alert][state.IsActive()]++
// }
for notification := range ackByNotificationCounts {
ts := opentsdb.TagSet{"notification": notification}
err := collect.Put("alerts.acknowledgement_status_by_notification",
ts.Copy().Merge(opentsdb.TagSet{"status": "unacknowledged"}),
ackByNotificationCounts[notification][true])
if err != nil {
slog.Errorln(err)
}
err = collect.Put("alerts.acknowledgement_status_by_notification",
ts.Copy().Merge(opentsdb.TagSet{"status": "acknowledged"}),
ackByNotificationCounts[notification][false])
if err != nil {
slog.Errorln(err)
}
}
for notification, timeStamp := range unAckOldestByNotification {
ts := opentsdb.TagSet{"notification": notification}
var ago time.Duration
if !timeStamp.Equal(time.Unix(1<<63-62135596801, 999999999)) {
ago = utcNow().Sub(timeStamp)
}
err := collect.Put("alerts.oldest_unacked_by_notification",
ts,
ago.Seconds())
if err != nil {
slog.Errorln(err)
}
}
for alertName := range severityCounts {
ts := opentsdb.TagSet{"alert": alertName}
// The tagset of the alert is not included because there is no way to
// store the string of a group in OpenTSBD in a parsable way. This is
// because any delimiter we chose could also be part of a tag key or tag
// value.
for severity := range severityCounts[alertName] {
err := collect.Put("alerts.current_severity",
ts.Copy().Merge(opentsdb.TagSet{"severity": severity}),
severityCounts[alertName][severity])
if err != nil {
slog.Errorln(err)
}
err = collect.Put("alerts.last_abnormal_severity",
ts.Copy().Merge(opentsdb.TagSet{"severity": severity}),
abnormalCounts[alertName][severity])
if err != nil {
slog.Errorln(err)
}
}
err := collect.Put("alerts.acknowledgement_status",
ts.Copy().Merge(opentsdb.TagSet{"status": "unacknowledged"}),
ackStatusCounts[alertName][true])
err = collect.Put("alerts.acknowledgement_status",
ts.Copy().Merge(opentsdb.TagSet{"status": "acknowledged"}),
ackStatusCounts[alertName][false])
if err != nil {
slog.Errorln(err)
}
err = collect.Put("alerts.active_status",
ts.Copy().Merge(opentsdb.TagSet{"status": "active"}),
activeStatusCounts[alertName][true])
if err != nil {
slog.Errorln(err)
}
err = collect.Put("alerts.active_status",
ts.Copy().Merge(opentsdb.TagSet{"status": "inactive"}),
activeStatusCounts[alertName][false])
if err != nil {
slog.Errorln(err)
}
}
}
func c_cisco_ios(host, community string, cpuIntegrator tsIntegrator) (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
ts := opentsdb.TagSet{"host": host}
// CPU
if err := ciscoCPU(host, community, ts, cpuIntegrator, &md); err != nil {
return md, err
}
// ÎMemory
memRaw, err := snmp_subtree(host, community, ciscoBaseOID+ciscoMemoryPoolTable)
if err != nil {
return md, fmt.Errorf("failed to get ciscoMemoryPoolTable for host %v: %v", host, err)
}
idToPoolEntry := make(map[string]*ciscoMemoryPoolEntry)
for id, value := range memRaw {
sp := strings.SplitN(id, ".", 2)
if len(sp) != 2 {
slog.Errorln("unexpected length of snmp sub OID (%v) for ciscoMemoryPoolTable for host %v: %v", id, host)
}
columnID := sp[0]
entryID := sp[1]
if _, ok := idToPoolEntry[entryID]; !ok {
idToPoolEntry[entryID] = &ciscoMemoryPoolEntry{}
}
switch columnID {
case "2":
if v, ok := value.([]byte); ok {
if m, ok := idToPoolEntry[entryID]; ok {
m.PoolType = string(v)
} else {
slog.Errorf("failed to find cisco memory pool entry for entry id %v on host %v for memory pool type", entryID, host)
}
} else {
slog.Errorf("failed to convert memory pool label %v to []byte for host %v", value, host)
}
case "5":
if v, ok := value.(int64); ok {
if m, ok := idToPoolEntry[entryID]; ok {
m.Used = v
} else {
slog.Errorf("failed to find cisco memory pool entry for entry id %v on host %v for used memory", entryID, host)
}
} else {
slog.Errorf("failed to convert used memory value %v to int64 for host %v", value, host)
}
case "6":
if v, ok := value.(int64); ok {
if m, ok := idToPoolEntry[entryID]; ok {
m.Free = v
} else {
slog.Errorf("failed to find cisco memory pool entry for entry id %v on host %v for free memory", entryID, host)
}
} else {
slog.Errorf("failed to convert used memory value %v to int64 for host %v", value, host)
}
}
}
var totalFreeMem int64
var totalUsedMem int64
for _, entry := range idToPoolEntry {
ts := ts.Copy().Merge(opentsdb.TagSet{"name": entry.PoolType})
Add(&md, "cisco.mem.used", entry.Used, ts, metadata.Gauge, metadata.Bytes, ciscoMemoryPoolUsedDesc)
Add(&md, "cisco.mem.free", entry.Free, ts, metadata.Gauge, metadata.Bytes, ciscoMemoryPoolFreeDesc)
totalFreeMem += entry.Free
totalUsedMem += entry.Used
}
Add(&md, osMemFree, totalFreeMem, ts, metadata.Gauge, metadata.Bytes, osMemFreeDesc)
Add(&md, osMemUsed, totalUsedMem, ts, metadata.Gauge, metadata.Bytes, osMemUsedDesc)
totalMem := totalFreeMem + totalUsedMem
Add(&md, osMemTotal, totalMem, ts, metadata.Gauge, metadata.Bytes, osMemTotalDesc)
Add(&md, osMemPctFree, int64(float64(totalFreeMem)/float64(totalMem)*100), ts, metadata.Gauge, metadata.Pct, osMemPctFreeDesc)
return md, nil
}
func (c *Conf) loadLookup(s *parse.SectionNode) {
name := s.Name.Text
if _, ok := c.Lookups[name]; ok {
c.errorf("duplicate lookup name: %s", name)
}
l := Lookup{
Name: name,
}
l.Text = s.RawText
var lookupTags opentsdb.TagSet
saw := make(map[string]bool)
for _, n := range s.Nodes.Nodes {
c.at(n)
switch n := n.(type) {
case *parse.SectionNode:
if n.SectionType.Text != "entry" {
c.errorf("unexpected subsection type")
}
tags, err := opentsdb.ParseTags(n.Name.Text)
if tags == nil && err != nil {
c.error(err)
}
if _, ok := saw[tags.String()]; ok {
c.errorf("duplicate entry")
}
saw[tags.String()] = true
if len(tags) == 0 {
c.errorf("lookup entries require tags")
}
empty := make(opentsdb.TagSet)
for k := range tags {
empty[k] = ""
}
if len(lookupTags) == 0 {
lookupTags = empty
for k := range empty {
l.Tags = append(l.Tags, k)
}
} else if !lookupTags.Equal(empty) {
c.errorf("lookup tags mismatch, expected %v", lookupTags)
}
e := Entry{
Def: n.RawText,
Name: n.Name.Text,
ExprEntry: &ExprEntry{
AlertKey: models.NewAlertKey("", tags),
Values: make(map[string]string),
},
}
for _, en := range n.Nodes.Nodes {
c.at(en)
switch en := en.(type) {
case *parse.PairNode:
e.Values[en.Key.Text] = en.Val.Text
default:
c.errorf("unexpected node")
}
}
l.Entries = append(l.Entries, &e)
default:
c.errorf("unexpected node")
}
}
c.at(s)
c.Lookups[name] = &l
}
func (to *TagOverride) ApplyTags(t opentsdb.TagSet) {
if to.tags != nil {
t = t.Merge(to.tags)
}
}
func (s *Schedule) Host(filter string) (map[string]*HostData, error) {
timeFilterAge := time.Hour * 2 * 24
hosts := make(map[string]*HostData)
allHosts, err := s.Search.TagValuesByTagKey("host", timeFilterAge)
if err != nil {
return nil, err
}
for _, h := range allHosts {
hosts[h] = newHostData()
}
states := s.GetOpenStates()
silences := s.Silenced()
// These are all fetched by metric since that is how we store it in redis,
// so this makes for the fastest response
tagsByKey := func(metric, hostKey string) (map[string][]opentsdb.TagSet, error) {
byKey := make(map[string][]opentsdb.TagSet)
tags, err := s.Search.FilteredTagSets(metric, nil)
if err != nil {
return byKey, err
}
for _, ts := range tags {
if host, ok := ts[hostKey]; ok {
// Make sure the host exists based on our time filter
if _, ok := hosts[host]; ok {
byKey[host] = append(byKey[host], ts)
}
}
}
return byKey, nil
}
oldTimestamp := time.Now().Add(-timeFilterAge).Unix()
oldOrErr := func(ts int64, err error) bool {
if ts < oldTimestamp || err != nil {
return true
}
return false
}
osNetBytesTags, err := tagsByKey("os.net.bytes", "host")
if err != nil {
return nil, err
}
osNetVirtualBytesTags, err := tagsByKey("os.net.virtual.bytes", "host")
if err != nil {
return nil, err
}
osNetBondBytesTags, err := tagsByKey("os.net.bond.bytes", "host")
if err != nil {
return nil, err
}
osNetTunnelBytesTags, err := tagsByKey("os.net.tunnel.bytes", "host")
if err != nil {
return nil, err
}
osNetOtherBytesTags, err := tagsByKey("os.net.other.bytes", "host")
if err != nil {
return nil, err
}
osNetIfSpeedTags, err := tagsByKey("os.net.ifspeed", "host")
if err != nil {
return nil, err
}
osNetVirtualIfSpeedTags, err := tagsByKey("os.net.virtual.ifspeed", "host")
if err != nil {
return nil, err
}
osNetBondIfSpeedTags, err := tagsByKey("os.net.bond.ifspeed", "host")
if err != nil {
return nil, err
}
osNetTunnelIfSpeedTags, err := tagsByKey("os.net.tunnel.ifspeed", "host")
if err != nil {
return nil, err
}
osNetOtherIfSpeedTags, err := tagsByKey("os.net.other.ifspeed", "host")
if err != nil {
return nil, err
}
hwChassisTags, err := tagsByKey("hw.chassis", "host")
if err != nil {
return nil, err
}
hwPhysicalDiskTags, err := tagsByKey("hw.storage.pdisk", "host")
if err != nil {
return nil, err
}
hwVirtualDiskTags, err := tagsByKey("hw.storage.vdisk", "host")
if err != nil {
return nil, err
}
hwControllersTags, err := tagsByKey("hw.storage.controller", "host")
if err != nil {
return nil, err
}
hwBatteriesTags, err := tagsByKey("hw.storage.battery", "host")
if err != nil {
return nil, err
}
hwPowerSuppliesTags, err := tagsByKey("hw.ps", "host")
if err != nil {
return nil, err
}
hwTempsTags, err := tagsByKey("hw.chassis.temps.reading", "host")
if err != nil {
return nil, err
}
hwBoardPowerTags, err := tagsByKey("hw.chassis.power.reading", "host")
if err != nil {
return nil, err
}
diskTags, err := tagsByKey("os.disk.fs.space_total", "host")
if err != nil {
return nil, err
}
serviceTags, err := tagsByKey("os.service.running", "host")
if err != nil {
return nil, err
}
// Will assume the same tagsets exist .mem_real, .mem_virtual and possibly .count
processTags, err := tagsByKey("os.proc.cpu", "host")
if err != nil {
return nil, err
}
// Will make the assumption that the metric bosun.ping.timeout, resolved, and rtt
// all share the same tagset
icmpTimeOutTags, err := tagsByKey("bosun.ping.timeout", "dst_host")
if err != nil {
return nil, err
}
for name, host := range hosts {
host.Name = name
hostTagSet := opentsdb.TagSet{"host": host.Name}
hostMetadata, err := s.GetMetadata("", hostTagSet)
if err != nil {
slog.Error(err)
}
processHostIncidents(host, states, silences)
for _, ts := range icmpTimeOutTags[host.Name] {
// The host tag represents the polling source for these set of metrics
source, ok := ts["host"]
if !ok {
slog.Errorf("couldn't find source tag for icmp data for host %s", host.Name)
}
// 1 Means it timed out
timeout, timestamp, err := s.Search.GetLast("bosun.ping.timeout", ts.String(), false)
if oldOrErr(timestamp, err) {
continue
}
rtt, rttTimestamp, _ := s.Search.GetLast("bosun.ping.rtt", ts.String(), false)
// 1 means dns resolution was successful
dnsLookup, dnsTimestamp, dnsErr := s.Search.GetLast("bosun.ping.resolved", ts.String(), false)
host.ICMPData[source] = &ICMPData{
TimedOut: timeout == 1 && err == nil,
TimedOutLastUpdated: timestamp,
DNSResolved: dnsLookup == 1 && dnsErr == nil,
DNSResolvedLastUpdated: dnsTimestamp,
RTTMS: rtt,
RTTLastUpdated: rttTimestamp,
}
}
for _, ts := range serviceTags[host.Name] {
name, ok := ts["name"]
if !ok {
slog.Errorf("couldn't find service name tag %s for host %s", name, host.Name)
continue
}
fstatus, timestamp, err := s.Search.GetLast("os.service.running", ts.String(), false)
running := false
if fstatus != 0 {
running = true
}
if !oldOrErr(timestamp, err) {
host.Services[name] = &ServiceStatus{
Running: running,
RunningLastUpdated: timestamp,
}
}
}
for _, ts := range processTags[host.Name] {
name, ok := ts["name"]
if !ok {
slog.Errorf("couldn't find process name tag %s for host %s", name, host.Name)
continue
}
p := &Process{}
p.CPUPercentUsed, p.CPUPercentLastUpdated, err = s.Search.GetLast("os.proc.cpu", ts.String(), true)
if oldOrErr(p.CPUPercentLastUpdated, err) {
continue
}
p.UsedRealBytes, p.UsedRealBytesLastUpdated, _ = s.Search.GetLastInt64("os.proc.mem.real", ts.String(), false)
p.UsedVirtualBytes, p.UsedVirtualBytesLastUpdated, _ = s.Search.GetLastInt64("os.proc.mem.virtual", ts.String(), false)
p.Count, p.CountLastUpdated, _ = s.Search.GetLastInt64("os.proc.count", ts.String(), false)
host.Processes[name] = p
}
// Process Hardware Chassis States
for _, ts := range hwChassisTags[host.Name] {
component, ok := ts["component"]
if !ok {
return nil, fmt.Errorf("couldn't find component tag for host %s", host.Name)
}
fstatus, timestamp, err := s.Search.GetLast("hw.chassis", ts.String(), false)
if !oldOrErr(timestamp, err) {
host.Hardware.ChassisComponents[component] = &ChassisComponent{
Status: statusString(int64(fstatus), 0, "Ok", "Bad"),
StatusLastUpdated: timestamp,
}
}
}
for _, ts := range hwTempsTags[host.Name] {
name, ok := ts["name"]
if !ok {
slog.Errorf("couldn't find name tag %s for host %s", name, host.Name)
}
t := &Temp{}
var tStatus float64
tStatus, t.StatusLastUpdated, err = s.Search.GetLast("hw.chassis.temps", ts.String(), false)
t.Celsius, t.CelsiusLastUpdated, _ = s.Search.GetLast("hw.chassis.temps.reading", ts.String(), false)
if oldOrErr(t.StatusLastUpdated, err) {
continue
}
t.Status = statusString(int64(tStatus), 0, "Ok", "Bad")
host.Hardware.Temps[name] = t
}
for _, ts := range hwPowerSuppliesTags[host.Name] {
id, ok := ts["id"]
if !ok {
return nil, fmt.Errorf("couldn't find power supply tag for host %s", host.Name)
}
idPlus, err := strconv.Atoi(id)
if err != nil {
slog.Errorf("couldn't conver it do integer for power supply id %s", id)
}
idPlus++
ps := &PowerSupply{}
fstatus, timestamp, err := s.Search.GetLast("hw.ps", ts.String(), false)
ps.Amps, ps.AmpsLastUpdated, _ = s.Search.GetLast("hw.chassis.current.reading", opentsdb.TagSet{"host": host.Name, "id": fmt.Sprintf("PS%v", idPlus)}.String(), false)
ps.Volts, ps.VoltsLastUpdated, _ = s.Search.GetLast("hw.chassis.volts.reading", opentsdb.TagSet{"host": host.Name, "name": fmt.Sprintf("PS%v_Voltage_%v", idPlus, idPlus)}.String(), false)
if oldOrErr(timestamp, err) {
continue
}
ps.Status = statusString(int64(fstatus), 0, "Ok", "Bad")
host.Hardware.PowerSupplies[id] = ps
for _, m := range hostMetadata {
if m.Name != "psMeta" || m.Time.Before(time.Now().Add(-timeFilterAge)) || !m.Tags.Equal(ts) {
continue
}
if val, ok := m.Value.(string); ok {
err = json.Unmarshal([]byte(val), &ps)
if err != nil {
slog.Errorf("error unmarshalling power supply meta for host %s, while generating host api: %s", host.Name, err)
} else {
host.Hardware.PowerSupplies[id] = ps
}
}
}
}
for _, ts := range hwBatteriesTags[host.Name] {
id, ok := ts["id"]
if !ok {
slog.Errorf("couldn't find battery id tag %s for host %s", id, host.Name)
continue
}
fstatus, timestamp, err := s.Search.GetLast("hw.storage.battery", ts.String(), false)
if !oldOrErr(timestamp, err) {
host.Hardware.Storage.Batteries[id] = &Battery{
Status: statusString(int64(fstatus), 0, "Ok", "Bad"),
StatusLastUpdated: timestamp,
}
}
}
for _, ts := range hwBoardPowerTags[host.Name] {
fstatus, timestamp, err := s.Search.GetLast("hw.chassis.power.reading", ts.String(), false)
if !oldOrErr(timestamp, err) {
host.Hardware.BoardPowerReading = &BoardPowerReading{
Watts: int64(fstatus),
WattsLastUpdated: timestamp,
}
}
}
for _, ts := range hwPhysicalDiskTags[host.Name] {
id, ok := ts["id"]
if !ok {
return nil, fmt.Errorf("couldn't find physical disk id tag for host %s", host.Name)
}
pd := &PhysicalDisk{}
fstatus, timestamp, err := s.Search.GetLast("hw.storage.pdisk", ts.String(), false)
if !oldOrErr(timestamp, err) {
pd.Status = statusString(int64(fstatus), 0, "Ok", "Bad")
pd.StatusLastUpdated = timestamp
host.Hardware.Storage.PhysicalDisks[id] = pd
}
for _, m := range hostMetadata {
if m.Name != "physicalDiskMeta" || m.Time.Before(time.Now().Add(-timeFilterAge)) || !m.Tags.Equal(ts) {
continue
}
if val, ok := m.Value.(string); ok {
err = json.Unmarshal([]byte(val), &pd)
if err != nil {
slog.Errorf("error unmarshalling addresses for host %s, interface %s while generating host api: %s", host.Name, m.Tags["iface"], err)
} else {
host.Hardware.Storage.PhysicalDisks[id] = pd
}
}
}
}
for _, ts := range hwVirtualDiskTags[host.Name] {
id, ok := ts["id"]
if !ok {
return nil, fmt.Errorf("couldn't find virtual disk id tag for host %s", host.Name)
}
fstatus, timestamp, err := s.Search.GetLast("hw.storage.vdisk", ts.String(), false)
if !oldOrErr(timestamp, err) {
host.Hardware.Storage.VirtualDisks[id] = &VirtualDisk{
Status: statusString(int64(fstatus), 0, "Ok", "Bad"),
StatusLastUpdated: timestamp,
}
}
}
for _, ts := range hwControllersTags[host.Name] {
id, ok := ts["id"]
if !ok {
return nil, fmt.Errorf("couldn't find controller id tag for host %s", host.Name)
}
fstatus, timestamp, err := s.Search.GetLast("hw.storage.controller", ts.String(), false)
c := &Controller{}
if !oldOrErr(timestamp, err) {
c.Status = statusString(int64(fstatus), 0, "Ok", "Bad")
c.StatusLastUpdated = timestamp
host.Hardware.Storage.Controllers[id] = c
}
for _, m := range hostMetadata {
if m.Name != "controllerMeta" || m.Time.Before(time.Now().Add(-timeFilterAge)) || !m.Tags.Equal(ts) {
continue
}
if val, ok := m.Value.(string); ok {
err = json.Unmarshal([]byte(val), &c)
if err != nil {
slog.Errorf("error unmarshalling controller meta for host %s: %s", host.Name, err)
} else {
host.Hardware.Storage.Controllers[id] = c
}
}
}
}
for _, ts := range diskTags[host.Name] {
disk, ok := ts["disk"]
if !ok {
return nil, fmt.Errorf("couldn't find disk tag for host %s", host.Name)
}
d := &Disk{}
d.TotalBytes, d.StatsLastUpdated, err = s.Search.GetLastInt64("os.disk.fs.space_total", ts.String(), false)
d.UsedBytes, _, _ = s.Search.GetLastInt64("os.disk.fs.space_used", ts.String(), false)
if oldOrErr(d.StatsLastUpdated, err) {
continue
}
host.Disks[disk] = d
for _, m := range hostMetadata {
if m.Name != "label" || m.Time.Before(time.Now().Add(-timeFilterAge)) || !m.Tags.Equal(ts) {
continue
}
if label, ok := m.Value.(string); ok {
host.Disks[disk].Label = label
break
}
}
}
// Get CPU, Memory, Uptime
var timestamp int64
var cpu float64
if cpu, timestamp, err = s.Search.GetLast("os.cpu", hostTagSet.String(), true); err != nil {
cpu, timestamp, _ = s.Search.GetLast("cisco.cpu", hostTagSet.String(), false)
}
host.CPU.PercentUsed = cpu
host.CPU.StatsLastUpdated = timestamp
host.Memory.TotalBytes, host.Memory.StatsLastUpdated, _ = s.Search.GetLast("os.mem.total", hostTagSet.String(), false)
host.Memory.UsedBytes, _, _ = s.Search.GetLast("os.mem.used", hostTagSet.String(), false)
host.UptimeSeconds, _, _ = s.Search.GetLastInt64("os.system.uptime", hostTagSet.String(), false)
for _, m := range hostMetadata {
if m.Time.Before(time.Now().Add(-timeFilterAge)) {
continue
}
var iface *HostInterface
if name := m.Tags["iface"]; name != "" {
if host.Interfaces[name] == nil {
h := new(HostInterface)
host.Interfaces[name] = h
}
iface = host.Interfaces[name]
}
if name := m.Tags["iname"]; name != "" && iface != nil {
iface.Name = name
}
switch val := m.Value.(type) {
case string:
switch m.Name {
case "addresses":
if iface != nil {
addresses := []string{}
err = json.Unmarshal([]byte(val), &addresses)
if err != nil {
slog.Errorf("error unmarshalling addresses for host %s, interface %s while generating host api: %s", host.Name, m.Tags["iface"], err)
}
for _, address := range addresses {
iface.IPAddresses = append(iface.IPAddresses, address)
}
}
case "cdpCacheEntries":
if iface != nil {
var cdpCacheEntries CDPCacheEntries
err = json.Unmarshal([]byte(val), &cdpCacheEntries)
if err != nil {
slog.Errorf("error unmarshalling cdpCacheEntries for host %s, interface %s while generating host api: %s", host.Name, m.Tags["iface"], err)
} else {
iface.CDPCacheEntries = cdpCacheEntries
}
}
case "remoteMacs":
if iface != nil {
remoteMacs := []string{}
err = json.Unmarshal([]byte(val), &remoteMacs)
if err != nil {
slog.Errorf("error unmarshalling remoteMacs for host %s, interface %s while generating host api: %s", host.Name, m.Tags["iface"], err)
} else {
iface.RemoteMacs = remoteMacs
}
}
case "description", "alias":
if iface != nil {
iface.Description = val
}
case "dataStores":
dataStores := []string{}
err = json.Unmarshal([]byte(val), &dataStores)
if err != nil {
slog.Errorf("error unmarshalling datastores for host %s while generating host api: %s", host.Name, err)
}
for _, dataStore := range dataStores {
tags := opentsdb.TagSet{"disk": dataStore}.String()
total, totalTs, totalErr := s.Search.GetLastInt64("vsphere.disk.space_total", tags, false)
used, usedTs, usedErr := s.Search.GetLastInt64("vsphere.disk.space_used", tags, false)
if totalErr != nil || usedErr != nil || totalTs < 1 || usedTs < 1 {
continue
}
host.Disks[dataStore] = &Disk{
TotalBytes: total,
UsedBytes: used,
}
}
case "mac":
if iface != nil {
iface.MAC = val
}
case "manufacturer":
host.Manufacturer = val
case "master":
if iface != nil {
iface.Master = val
}
case "memory":
if name := m.Tags["name"]; name != "" {
fstatus, timestamp, err := s.Search.GetLast("hw.chassis.memory", opentsdb.TagSet{"host": host.Name, "name": name}.String(), false)
// Status code uses the severity function in collectors/dell_hw.go. That is a binary
// state that is 0 for non-critical or Ok. Todo would be to update this with more
// complete status codes when HW collector is refactored and we have something to
// clean out addr entries from the tagset metadata db
host.Hardware.Memory[name] = &MemoryModule{
StatusLastUpdated: timestamp,
Size: val,
}
// Only set if we have a value
if err == nil && timestamp > 0 {
host.Hardware.Memory[name].Status = statusString(int64(fstatus), 0, "Ok", "Bad")
}
}
case "hypervisor":
host.VM = &VM{}
host.VM.Host = val
powerstate, timestamp, err := s.Search.GetLast("vsphere.guest.powered_state", opentsdb.TagSet{"guest": host.Name}.String(), false)
if timestamp > 0 && err != nil {
switch int64(powerstate) {
case 0:
host.VM.PowerState = "poweredOn"
case 1:
host.VM.PowerState = "poweredOff"
case 2:
host.VM.PowerState = "suspended"
}
host.VM.PowerStateLastUpdated = timestamp
}
if hostsHost, ok := hosts[val]; ok {
hostsHost.Guests = append(hostsHost.Guests, host.Name)
}
case "model":
host.Model = val
case "name":
if iface != nil {
iface.Name = val
}
case "processor":
if name := m.Tags["name"]; name != "" {
host.CPU.Processors[name] = val
}
case "serialNumber":
host.SerialNumber = val
case "version":
host.OS.Version = val
case "versionCaption", "uname":
host.OS.Caption = val
}
case float64:
switch m.Name {
case "speed":
if iface != nil {
iface.LinkSpeed = int64(val)
}
}
}
}
GetIfaceBits := func(netType string, ifaceId string, iface *HostInterface, host string, tags []opentsdb.TagSet) error {
metric := "os.net." + netType + ".bytes"
if netType == "" {
metric = "os.net.bytes"
}
for _, ts := range tags {
if ts["iface"] != ifaceId {
continue
}
dir, ok := ts["direction"]
if !ok {
continue
}
val, timestamp, _ := s.Search.GetLastInt64(metric, ts.String(), true)
if dir == "in" {
iface.Inbps = val * 8
}
if dir == "out" {
iface.Outbps = val * 8
}
iface.StatsLastUpdated = timestamp
iface.Type = netType
}
return nil
}
GetIfaceSpeed := func(netType string, ifaceId string, iface *HostInterface, host string, tags []opentsdb.TagSet) error {
metric := "os.net." + netType + ".ifspeed"
if netType == "" {
metric = "os.net.ifspeed"
}
for _, ts := range tags {
if ts["iface"] != ifaceId {
continue
}
val, timestamp, err := s.Search.GetLastInt64(metric, ts.String(), false)
if !oldOrErr(timestamp, err) {
iface.LinkSpeed = val
}
}
return nil
}
for ifaceId, iface := range host.Interfaces {
if err := GetIfaceBits("", ifaceId, iface, host.Name, osNetBytesTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceBits("virtual", ifaceId, iface, host.Name, osNetVirtualBytesTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceBits("bond", ifaceId, iface, host.Name, osNetBondBytesTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceBits("tunnel", ifaceId, iface, host.Name, osNetTunnelBytesTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceBits("other", ifaceId, iface, host.Name, osNetOtherBytesTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceSpeed("", ifaceId, iface, host.Name, osNetIfSpeedTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceSpeed("virtual", ifaceId, iface, host.Name, osNetVirtualIfSpeedTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceSpeed("bond", ifaceId, iface, host.Name, osNetBondIfSpeedTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceSpeed("tunnel", ifaceId, iface, host.Name, osNetTunnelIfSpeedTags[host.Name]); err != nil {
return nil, err
}
if err := GetIfaceSpeed("other", ifaceId, iface, host.Name, osNetOtherIfSpeedTags[host.Name]); err != nil {
return nil, err
}
}
host.Clean()
}
return hosts, nil
}
func DatabaseParseResults(c conf.Database, query *DatabaseQuery, results *[][]string) (opentsdb.MultiDataPoint, error) {
if len(*results) < 1 {
return nil, nil
}
numTagLoops := len((*results)[0]) - 1
if query.hasTime {
numTagLoops--
}
if numTagLoops < 0 {
return nil, fmt.Errorf("invalid number of columns")
}
numTagLoops = (numTagLoops - (numTagLoops % 2)) / 2
var md opentsdb.MultiDataPoint
var tagName string
var tagValue string
var tagsBase opentsdb.TagSet
baseName := c.Type + "."
if c.DBName == "" {
tagsBase = opentsdb.TagSet{"inst_id": strconv.Itoa(c.InstId)}
} else {
tagsBase = opentsdb.TagSet{"db_name": c.DBName, "inst_id": strconv.Itoa(c.InstId)}
}
for _, result := range *results {
if result[0] == "" {
continue
}
// Check result[0] is vaild metric value?
tags := tagsBase.Copy()
for i := 0; i < numTagLoops; i++ {
if query.hasTime {
tagName = strings.Replace(result[(i*2)+2], " ", "_", -1)
tagValue = strings.Replace(result[(i*2)+3], " ", "_", -1)
} else {
tagName = strings.Replace(result[(i*2)+1], " ", "_", -1)
tagValue = strings.Replace(result[(i*2)+2], " ", "_", -1)
}
tagName, _ = opentsdb.Clean(tagName)
tagValue, _ = opentsdb.Clean(tagValue)
tagName = ContinuousCollectorVars.reTwoOrMoreUnderscore.ReplaceAllString(tagName, "_")
tagValue = ContinuousCollectorVars.reTwoOrMoreUnderscore.ReplaceAllString(tagValue, "_")
if tagName == "" || tagValue == "" {
continue
}
tags = tags.Merge(opentsdb.TagSet{tagName: tagValue})
}
// Add metadata tag set type to configuration file?
if query.hasTime {
timestamp, _ := strconv.ParseInt(result[1], 10, 64)
if timestamp < 1 {
return nil, fmt.Errorf("invalid timestamp")
}
AddTS(&md, baseName+query.name, timestamp, result[0], tags, metadata.Unknown, metadata.None, query.description)
} else {
Add(&md, baseName+query.name, result[0], tags, metadata.Unknown, metadata.None, query.description)
}
}
return md, nil
}
// CollectStates sends various state information to bosun with collect.
func (s *Schedule) CollectStates() {
// [AlertName][Severity]Count
severityCounts := make(map[string]map[string]int64)
abnormalCounts := make(map[string]map[string]int64)
ackStatusCounts := make(map[string]map[bool]int64)
activeStatusCounts := make(map[string]map[bool]int64)
// Initalize the Counts
for _, alert := range s.Conf.Alerts {
severityCounts[alert.Name] = make(map[string]int64)
abnormalCounts[alert.Name] = make(map[string]int64)
var i Status
for i = 1; i.String() != "none"; i++ {
severityCounts[alert.Name][i.String()] = 0
abnormalCounts[alert.Name][i.String()] = 0
}
ackStatusCounts[alert.Name] = make(map[bool]int64)
activeStatusCounts[alert.Name] = make(map[bool]int64)
ackStatusCounts[alert.Name][false] = 0
activeStatusCounts[alert.Name][false] = 0
ackStatusCounts[alert.Name][true] = 0
activeStatusCounts[alert.Name][true] = 0
}
for _, state := range s.status {
if !state.Open {
continue
}
severity := state.Status().String()
lastAbnormal := state.AbnormalStatus().String()
severityCounts[state.Alert][severity]++
abnormalCounts[state.Alert][lastAbnormal]++
ackStatusCounts[state.Alert][state.NeedAck]++
activeStatusCounts[state.Alert][state.IsActive()]++
}
for alertName := range severityCounts {
ts := opentsdb.TagSet{"alert": alertName}
// The tagset of the alert is not included because there is no way to
// store the string of a group in OpenTSBD in a parsable way. This is
// because any delimiter we chose could also be part of a tag key or tag
// value.
for severity := range severityCounts[alertName] {
err := collect.Put("alerts.current_severity",
ts.Copy().Merge(opentsdb.TagSet{"severity": severity}),
severityCounts[alertName][severity])
if err != nil {
slog.Errorln(err)
}
err = collect.Put("alerts.last_abnormal_severity",
ts.Copy().Merge(opentsdb.TagSet{"severity": severity}),
abnormalCounts[alertName][severity])
if err != nil {
slog.Errorln(err)
}
}
err := collect.Put("alerts.acknowledgement_status",
ts.Copy().Merge(opentsdb.TagSet{"status": "unacknowledged"}),
ackStatusCounts[alertName][true])
err = collect.Put("alerts.acknowledgement_status",
ts.Copy().Merge(opentsdb.TagSet{"status": "acknowledged"}),
ackStatusCounts[alertName][false])
if err != nil {
slog.Errorln(err)
}
err = collect.Put("alerts.active_status",
ts.Copy().Merge(opentsdb.TagSet{"status": "active"}),
activeStatusCounts[alertName][true])
if err != nil {
slog.Errorln(err)
}
err = collect.Put("alerts.active_status",
ts.Copy().Merge(opentsdb.TagSet{"status": "inactive"}),
activeStatusCounts[alertName][false])
if err != nil {
slog.Errorln(err)
}
}
}
func c_varnish_unix() (opentsdb.MultiDataPoint, error) {
var md opentsdb.MultiDataPoint
const metric = "varnish."
r, err := util.Command(5*time.Second, nil, "varnishstat", "-j")
if err != nil {
return nil, err
}
var stats varnishStats
if err := json.NewDecoder(r).Decode(&stats); err != nil {
return nil, err
}
for name, raw := range stats {
if name == "timestamp" {
continue
}
var v varnishStat
if err := json.Unmarshal(raw, &v); err != nil {
slog.Errorln("varnish parser error:", name, err)
continue
}
ts := opentsdb.TagSet{}
// special case for backend stats. extract backend name, host and port, put
// them in tags and remove them in name.
// the format is like "name(host,,port)" for the "ident" field of "VBE" type
if v.Type == "VBE" {
subtype := v.SubType
name = strings.Replace(name, "."+subtype, "", -1)
idx := strings.Index(subtype, "(")
if idx < 0 || len(subtype)-idx < 4 {
// output format changed, ignore
continue
}
ss := strings.Split(subtype[idx+1:len(subtype)-1], ",")
if len(ss) != 3 {
// output format changed, ignore
continue
}
ts.Merge(opentsdb.TagSet{"backend": subtype[:idx]})
ts.Merge(opentsdb.TagSet{"endpoint": ss[0] + "_" + ss[2]})
}
rate := metadata.RateType(metadata.Gauge)
if flag := v.Flag; flag == "a" || flag == "c" {
rate = metadata.Counter
}
unit := metadata.Unit(metadata.Count)
if v.Format == "B" {
unit = metadata.Bytes
}
Add(&md, metric+strings.ToLower(name), v.Value, ts, rate, unit, v.Desc)
}
return md, nil
}
func rabbitmqMessageStats(p string, ts opentsdb.TagSet, ms rmqMessageStats) opentsdb.MultiDataPoint {
var md opentsdb.MultiDataPoint
Add(&md, p+"message_stats", ms.Ack, ts.Copy().Merge(opentsdb.TagSet{"method": "ack"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsAck)
Add(&md, p+"message_stats", ms.Confirm, ts.Copy().Merge(opentsdb.TagSet{"method": "confirm"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsConfirm)
Add(&md, p+"message_stats", ms.Deliver, ts.Copy().Merge(opentsdb.TagSet{"method": "deliver"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsDeliver)
Add(&md, p+"message_stats", ms.DeliverGet, ts.Copy().Merge(opentsdb.TagSet{"method": "deliver_get"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsDeliverGet)
Add(&md, p+"message_stats", ms.DeliverNoAck, ts.Copy().Merge(opentsdb.TagSet{"method": "deliver_noack"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsDeliverNoAck)
Add(&md, p+"message_stats", ms.Get, ts.Copy().Merge(opentsdb.TagSet{"method": "get"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsGet)
Add(&md, p+"message_stats", ms.GetNoAck, ts.Copy().Merge(opentsdb.TagSet{"method": "get_noack"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsGetNoack)
Add(&md, p+"message_stats", ms.Publish, ts.Copy().Merge(opentsdb.TagSet{"method": "publish"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsPublish)
Add(&md, p+"message_stats", ms.PublishIn, ts.Copy().Merge(opentsdb.TagSet{"method": "publish_in"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsPublishIn)
Add(&md, p+"message_stats", ms.PublishOut, ts.Copy().Merge(opentsdb.TagSet{"method": "publish_out"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsPublishOut)
Add(&md, p+"message_stats", ms.Redeliver, ts.Copy().Merge(opentsdb.TagSet{"method": "redeliver"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsRedeliver)
Add(&md, p+"message_stats", ms.Return, ts.Copy().Merge(opentsdb.TagSet{"method": "return"}),
metadata.Counter, metadata.Message, DescRmqMessageStatsReturn)
return md
}
func NewAlertKey(name string, group opentsdb.TagSet) AlertKey {
return AlertKey(name + group.String())
}
func tagMetaKey(tags opentsdb.TagSet, name string) string {
return fmt.Sprintf("tmeta:%s:%s", tags.Tags(), name)
}
