func makeChart(r opentsdb.ResponseSet, m_units map[string]string) ([]*chartSeries, error) {
var series []*chartSeries
for _, resp := range r {
dps := make([][2]float64, 0)
for k, v := range resp.DPS {
ki, err := strconv.ParseInt(k, 10, 64)
if err != nil {
return nil, err
}
dps = append(dps, [2]float64{float64(ki), float64(v)})
}
if len(dps) > 0 {
slice.Sort(dps, func(i, j int) bool {
return dps[i][0] < dps[j][0]
})
name := resp.Metric
if len(resp.Tags) > 0 {
name += resp.Tags.String()
}
series = append(series, &chartSeries{
Name: name,
Metric: resp.Metric,
Tags: resp.Tags,
Data: dps,
Unit: m_units[resp.Metric],
})
}
}
return series, nil
}
func (s *Schedule) ExprGraph(t miniprofiler.Timer, unit string, res []*expr.Result) (chart.Chart, error) {
c := chart.ScatterChart{
Key: chart.Key{Pos: "itl"},
YRange: chart.Range{Label: unit},
}
c.XRange.Time = true
for ri, r := range res {
rv := r.Value.(expr.Series)
pts := make([]chart.EPoint, len(rv))
idx := 0
for k, v := range rv {
pts[idx].X = float64(k.Unix())
pts[idx].Y = v
idx++
}
slice.Sort(pts, func(i, j int) bool {
return pts[i].X < pts[j].X
})
c.AddData(r.Group.String(), pts, chart.PlotStyleLinesPoints, Autostyle(ri))
}
return &c, nil
}
func (s *Schedule) MarshalGroups(T miniprofiler.Timer, filter string) (*StateGroups, error) {
var silenced map[expr.AlertKey]Silence
T.Step("Silenced", func(miniprofiler.Timer) {
silenced = s.Silenced()
})
var groups map[StateTuple]States
var err error
status := make(States)
t := StateGroups{
TimeAndDate: s.Conf.TimeAndDate,
}
s.Lock("MarshallGroups")
defer s.Unlock()
T.Step("Setup", func(miniprofiler.Timer) {
matches, err2 := makeFilter(filter)
if err2 != nil {
err = err2
return
}
for k, v := range s.status {
if !v.Open {
continue
}
a := s.Conf.Alerts[k.Name()]
if a == nil {
err = fmt.Errorf("unknown alert %s", k.Name())
return
}
if matches(s.Conf, a, v) {
status[k] = v
}
}
})
if err != nil {
return nil, err
}
T.Step("GroupStates", func(T miniprofiler.Timer) {
groups = status.GroupStates(silenced)
})
T.Step("groups", func(T miniprofiler.Timer) {
for tuple, states := range groups {
var grouped []*StateGroup
switch tuple.Status {
case StWarning, StCritical, StUnknown, StError:
var sets map[string]expr.AlertKeys
T.Step(fmt.Sprintf("GroupSets (%d): %v", len(states), tuple), func(T miniprofiler.Timer) {
sets = states.GroupSets()
})
for name, group := range sets {
g := StateGroup{
Active: tuple.Active,
Status: tuple.Status,
Silenced: tuple.Silenced,
Subject: fmt.Sprintf("%s - %s", tuple.Status, name),
}
for _, ak := range group {
st := s.status[ak].Copy()
// remove some of the larger bits of state to reduce wire size
st.Body = ""
st.EmailBody = []byte{}
if len(st.History) > 1 {
st.History = st.History[len(st.History)-1:]
}
g.Children = append(g.Children, &StateGroup{
Active: tuple.Active,
Status: tuple.Status,
Silenced: tuple.Silenced,
AlertKey: ak,
Alert: ak.Name(),
Subject: string(st.Subject),
Ago: marshalTime(st.Last().Time),
State: st,
})
}
if len(g.Children) == 1 && g.Children[0].Subject != "" {
g.Subject = g.Children[0].Subject
}
grouped = append(grouped, &g)
}
default:
continue
}
if tuple.NeedAck {
t.Groups.NeedAck = append(t.Groups.NeedAck, grouped...)
} else {
t.Groups.Acknowledged = append(t.Groups.Acknowledged, grouped...)
}
}
})
T.Step("sort", func(T miniprofiler.Timer) {
gsort := func(grp []*StateGroup) func(i, j int) bool {
return func(i, j int) bool {
a := grp[i]
b := grp[j]
if a.Active && !b.Active {
return true
} else if !a.Active && b.Active {
return false
}
if a.Status != b.Status {
return a.Status > b.Status
}
if a.AlertKey != b.AlertKey {
return a.AlertKey < b.AlertKey
}
return a.Subject < b.Subject
}
}
slice.Sort(t.Groups.NeedAck, gsort(t.Groups.NeedAck))
slice.Sort(t.Groups.Acknowledged, gsort(t.Groups.Acknowledged))
})
return &t, nil
}
func Rule(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
var from, to time.Time
var err error
if f := r.FormValue("from"); len(f) > 0 {
from, err = time.Parse(tsdbFormatSecs, f)
if err != nil {
return nil, err
}
}
if f := r.FormValue("to"); len(f) > 0 {
to, err = time.Parse(tsdbFormatSecs, f)
if err != nil {
return nil, err
}
}
intervals := 1
if i := r.FormValue("intervals"); len(i) > 0 {
intervals, err = strconv.Atoi(r.FormValue("intervals"))
if err != nil {
return nil, err
}
if intervals < 1 {
return nil, fmt.Errorf("must be > 0 intervals")
}
}
if fz, tz := from.IsZero(), to.IsZero(); fz && tz {
from = time.Now()
} else if fz && !tz {
return nil, fmt.Errorf("cannot specify to without from")
} else if !fz && tz && intervals > 1 {
return nil, fmt.Errorf("cannot specify intervals without from and to")
}
c, a, hash, err := buildConfig(r)
if err != nil {
return nil, err
}
ch := make(chan int)
errch := make(chan error, intervals)
resch := make(chan *ruleResult, intervals)
var wg sync.WaitGroup
diff := -from.Sub(to)
if intervals > 1 {
diff /= time.Duration(intervals - 1)
}
worker := func() {
wg.Add(1)
for interval := range ch {
t.Step(fmt.Sprintf("interval %v", interval), func(t miniprofiler.Timer) {
now := from.Add(diff * time.Duration(interval))
res, err := procRule(t, c, a, now, interval != 0, r.FormValue("email"), r.FormValue("template_group"))
resch <- res
errch <- err
})
}
defer wg.Done()
}
for i := 0; i < 20; i++ {
go worker()
}
for i := 0; i < intervals; i++ {
ch <- i
}
close(ch)
wg.Wait()
close(errch)
close(resch)
type Result struct {
Group models.AlertKey
Result *models.Event
}
type Set struct {
Critical, Warning, Normal int
Time string
Results []*Result `json:",omitempty"`
}
type History struct {
Time, EndTime time.Time
Status string
}
type Histories struct {
History []*History
}
ret := struct {
Errors []string `json:",omitempty"`
Warnings []string `json:",omitempty"`
Sets []*Set
AlertHistory map[models.AlertKey]*Histories
Body string `json:",omitempty"`
Subject string `json:",omitempty"`
Data interface{} `json:",omitempty"`
Hash string
}{
AlertHistory: make(map[models.AlertKey]*Histories),
Hash: hash,
}
for err := range errch {
if err == nil {
continue
}
ret.Errors = append(ret.Errors, err.Error())
}
for res := range resch {
if res == nil {
continue
}
set := Set{
Critical: len(res.Criticals),
Warning: len(res.Warnings),
Normal: len(res.Normals),
Time: res.Time.Format(tsdbFormatSecs),
}
if res.Data != nil {
ret.Body = res.Body
ret.Subject = res.Subject
ret.Data = res.Data
for k, v := range res.Result {
set.Results = append(set.Results, &Result{
Group: k,
Result: v,
})
}
slice.Sort(set.Results, func(i, j int) bool {
a := set.Results[i]
b := set.Results[j]
if a.Result.Status != b.Result.Status {
return a.Result.Status > b.Result.Status
}
return a.Group < b.Group
})
}
for k, v := range res.Result {
if ret.AlertHistory[k] == nil {
ret.AlertHistory[k] = new(Histories)
}
h := ret.AlertHistory[k]
h.History = append(h.History, &History{
Time: v.Time,
Status: v.Status.String(),
})
}
ret.Sets = append(ret.Sets, &set)
ret.Warnings = append(ret.Warnings, res.Warning...)
}
slice.Sort(ret.Sets, func(i, j int) bool {
return ret.Sets[i].Time < ret.Sets[j].Time
})
for _, histories := range ret.AlertHistory {
hist := histories.History
slice.Sort(hist, func(i, j int) bool {
return hist[i].Time.Before(hist[j].Time)
})
for i := 1; i < len(hist); i++ {
if i < len(hist)-1 && hist[i].Status == hist[i-1].Status {
hist = append(hist[:i], hist[i+1:]...)
i--
}
}
for i, h := range hist[:len(hist)-1] {
h.EndTime = hist[i+1].Time
}
histories.History = hist[:len(hist)-1]
}
return &ret, nil
}
// Graph takes an OpenTSDB request data structure and queries OpenTSDB. Use the
// json parameter to pass JSON. Use the b64 parameter to pass base64-encoded
// JSON.
func Graph(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
j := []byte(r.FormValue("json"))
if bs := r.FormValue("b64"); bs != "" {
b, err := base64.StdEncoding.DecodeString(bs)
if err != nil {
return nil, err
}
j = b
}
if len(j) == 0 {
return nil, fmt.Errorf("either json or b64 required")
}
oreq, err := opentsdb.RequestFromJSON(j)
if err != nil {
return nil, err
}
if ads_v := r.FormValue("autods"); ads_v != "" {
ads_i, err := strconv.Atoi(ads_v)
if err != nil {
return nil, err
}
if err := oreq.AutoDownsample(ads_i); err != nil {
return nil, err
}
}
ar := make(map[int]bool)
for _, v := range r.Form["autorate"] {
if i, err := strconv.Atoi(v); err == nil {
ar[i] = true
}
}
queries := make([]string, len(oreq.Queries))
var start, end string
if s, ok := oreq.Start.(string); ok && strings.Contains(s, "-ago") {
start = strings.TrimSuffix(s, "-ago")
}
if s, ok := oreq.End.(string); ok && strings.Contains(s, "-ago") {
end = strings.TrimSuffix(s, "-ago")
}
if start == "" && end == "" {
s, sok := oreq.Start.(int64)
e, eok := oreq.End.(int64)
if sok && eok {
start = fmt.Sprintf("%vs", e-s)
}
}
m_units := make(map[string]string)
for i, q := range oreq.Queries {
if ar[i] {
meta, err := schedule.MetadataMetrics(q.Metric)
if err != nil {
return nil, err
}
if meta == nil {
return nil, fmt.Errorf("no metadata for %s: cannot use auto rate", q)
}
if meta.Unit != "" {
m_units[q.Metric] = meta.Unit
}
if meta.Rate != "" {
switch meta.Rate {
case metadata.Gauge:
// ignore
case metadata.Rate:
q.Rate = true
case metadata.Counter:
q.Rate = true
q.RateOptions = opentsdb.RateOptions{
Counter: true,
ResetValue: 1,
}
default:
return nil, fmt.Errorf("unknown metadata rate: %s", meta.Rate)
}
}
}
queries[i] = fmt.Sprintf(`q("%v", "%v", "%v")`, q, start, end)
if !schedule.Conf.TSDBContext().Version().FilterSupport() {
if err := schedule.Search.Expand(q); err != nil {
return nil, err
}
}
}
var tr opentsdb.ResponseSet
b, _ := json.MarshalIndent(oreq, "", " ")
t.StepCustomTiming("tsdb", "query", string(b), func() {
h := schedule.Conf.TSDBHost
if h == "" {
err = fmt.Errorf("tsdbHost not set")
return
}
tr, err = oreq.Query(h)
})
if err != nil {
return nil, err
}
cs, err := makeChart(tr, m_units)
if err != nil {
return nil, err
}
if _, present := r.Form["png"]; present {
c := chart.ScatterChart{
Title: fmt.Sprintf("%v - %v", oreq.Start, queries),
}
c.XRange.Time = true
if min, err := strconv.ParseFloat(r.FormValue("min"), 64); err == nil {
c.YRange.MinMode.Fixed = true
c.YRange.MinMode.Value = min
}
if max, err := strconv.ParseFloat(r.FormValue("max"), 64); err == nil {
c.YRange.MaxMode.Fixed = true
c.YRange.MaxMode.Value = max
}
for ri, r := range cs {
pts := make([]chart.EPoint, len(r.Data))
for idx, v := range r.Data {
pts[idx].X = v[0]
pts[idx].Y = v[1]
}
slice.Sort(pts, func(i, j int) bool {
return pts[i].X < pts[j].X
})
c.AddData(r.Name, pts, chart.PlotStyleLinesPoints, sched.Autostyle(ri))
}
w.Header().Set("Content-Type", "image/svg+xml")
white := color.RGBA{0xff, 0xff, 0xff, 0xff}
const width = 800
const height = 600
s := svg.New(w)
s.Start(width, height)
s.Rect(0, 0, width, height, "fill: #ffffff")
sgr := svgg.AddTo(s, 0, 0, width, height, "", 12, white)
c.Plot(sgr)
s.End()
return nil, nil
}
return struct {
Queries []string
Series []*chartSeries
}{
queries,
cs,
}, nil
}
func (s *Schedule) MarshalGroups(T miniprofiler.Timer, filter string) (*StateGroups, error) {
t := StateGroups{
TimeAndDate: s.Conf.TimeAndDate,
Silenced: s.Silenced(),
}
s.TimeLock(T)
defer s.Unlock()
status := make(States)
matches, err := makeFilter(filter)
if err != nil {
return nil, err
}
for k, v := range s.status {
if !v.Open {
continue
}
a := s.Conf.Alerts[k.Name()]
if a == nil {
return nil, fmt.Errorf("unknown alert %s", k.Name())
}
if matches(s.Conf, a, v) {
status[k] = v
}
}
var groups map[StateTuple]States
T.Step("GroupStates", func(T miniprofiler.Timer) {
groups = status.GroupStates()
})
T.Step("groups", func(T miniprofiler.Timer) {
for tuple, states := range groups {
var grouped []*StateGroup
switch tuple.Status {
case StWarning, StCritical, StUnknown, StError:
var sets map[string]expr.AlertKeys
T.Step(fmt.Sprintf("GroupSets (%d): %v", len(states), tuple), func(T miniprofiler.Timer) {
sets = states.GroupSets()
})
for name, group := range sets {
g := StateGroup{
Active: tuple.Active,
Status: tuple.Status,
Subject: fmt.Sprintf("%s - %s", tuple.Status, name),
Len: len(group),
}
for _, ak := range group {
st := s.status[ak]
g.Children = append(g.Children, &StateGroup{
Active: tuple.Active,
Status: tuple.Status,
AlertKey: ak,
Alert: ak.Name(),
Subject: string(st.Subject),
Ago: marshalTime(st.Last().Time),
})
}
grouped = append(grouped, &g)
}
default:
continue
}
if tuple.NeedAck {
t.Groups.NeedAck = append(t.Groups.NeedAck, grouped...)
} else {
t.Groups.Acknowledged = append(t.Groups.Acknowledged, grouped...)
}
}
})
gsort := func(grp []*StateGroup) func(i, j int) bool {
return func(i, j int) bool {
a := grp[i]
b := grp[j]
if a.Active && !b.Active {
return true
} else if !a.Active && b.Active {
return false
}
if a.Status != b.Status {
return a.Status > b.Status
}
if a.AlertKey != b.AlertKey {
return a.AlertKey < b.AlertKey
}
return a.Subject < b.Subject
}
}
slice.Sort(t.Groups.NeedAck, gsort(t.Groups.NeedAck))
slice.Sort(t.Groups.Acknowledged, gsort(t.Groups.Acknowledged))
return &t, nil
}
func (s *Schedule) MarshalGroups(T miniprofiler.Timer, filter string) (*StateGroups, error) {
var silenced SilenceTester
T.Step("Silenced", func(miniprofiler.Timer) {
silenced = s.Silenced()
})
var groups map[StateTuple]States
var err error
status := make(States)
t := StateGroups{
TimeAndDate: s.Conf.TimeAndDate,
}
t.FailingAlerts, t.UnclosedErrors = s.getErrorCounts()
T.Step("Setup", func(miniprofiler.Timer) {
matches, err2 := makeFilter(filter)
if err2 != nil {
err = err2
return
}
status2, err2 := s.GetOpenStates()
if err2 != nil {
err = err2
return
}
for k, v := range status2 {
a := s.Conf.Alerts[k.Name()]
if a == nil {
slog.Errorf("unknown alert %s", k.Name())
continue
}
if matches(s.Conf, a, v) {
status[k] = v
}
}
})
if err != nil {
return nil, err
}
T.Step("GroupStates", func(T miniprofiler.Timer) {
groups = status.GroupStates(silenced)
})
T.Step("groups", func(T miniprofiler.Timer) {
for tuple, states := range groups {
var grouped []*StateGroup
switch tuple.Status {
case models.StWarning, models.StCritical, models.StUnknown:
var sets map[string]models.AlertKeys
T.Step(fmt.Sprintf("GroupSets (%d): %v", len(states), tuple), func(T miniprofiler.Timer) {
sets = states.GroupSets(s.Conf.MinGroupSize)
})
for name, group := range sets {
g := StateGroup{
Active: tuple.Active,
Status: tuple.Status,
CurrentStatus: tuple.CurrentStatus,
Silenced: tuple.Silenced,
Subject: fmt.Sprintf("%s - %s", tuple.Status, name),
}
for _, ak := range group {
st := status[ak]
st.Body = ""
st.EmailBody = nil
g.Children = append(g.Children, &StateGroup{
Active: tuple.Active,
Status: tuple.Status,
Silenced: tuple.Silenced,
AlertKey: ak,
Alert: ak.Name(),
Subject: string(st.Subject),
Ago: marshalTime(st.Last().Time),
State: st,
IsError: !s.AlertSuccessful(ak.Name()),
})
}
if len(g.Children) == 1 && g.Children[0].Subject != "" {
g.Subject = g.Children[0].Subject
}
grouped = append(grouped, &g)
}
default:
continue
}
if tuple.NeedAck {
t.Groups.NeedAck = append(t.Groups.NeedAck, grouped...)
} else {
t.Groups.Acknowledged = append(t.Groups.Acknowledged, grouped...)
}
}
})
T.Step("sort", func(T miniprofiler.Timer) {
gsort := func(grp []*StateGroup) func(i, j int) bool {
return func(i, j int) bool {
a := grp[i]
b := grp[j]
if a.Active && !b.Active {
return true
} else if !a.Active && b.Active {
return false
}
if a.Status != b.Status {
return a.Status > b.Status
}
if a.AlertKey != b.AlertKey {
return a.AlertKey < b.AlertKey
}
return a.Subject < b.Subject
}
}
slice.Sort(t.Groups.NeedAck, gsort(t.Groups.NeedAck))
slice.Sort(t.Groups.Acknowledged, gsort(t.Groups.Acknowledged))
})
return &t, nil
}