func (e *State) walkUnary(node *parse.UnaryNode, T miniprofiler.Timer) *Results {
a := e.walk(node.Arg, T)
T.Step("walkUnary: "+node.OpStr, func(T miniprofiler.Timer) {
for _, r := range a.Results {
if an, aok := r.Value.(Scalar); aok && math.IsNaN(float64(an)) {
r.Value = Scalar(math.NaN())
continue
}
switch rt := r.Value.(type) {
case Scalar:
r.Value = Scalar(uoperate(node.OpStr, float64(rt)))
case Number:
r.Value = Number(uoperate(node.OpStr, float64(rt)))
case Series:
s := make(Series)
for k, v := range rt {
s[k] = uoperate(node.OpStr, float64(v))
}
r.Value = s
default:
panic(ErrUnknownOp)
}
}
})
return a
}
func bandTSDB(e *State, T miniprofiler.Timer, query, duration, period string, num float64, rfunc func(*Results, *opentsdb.Response, time.Duration) error) (r *Results, err error) {
r = new(Results)
r.IgnoreOtherUnjoined = true
r.IgnoreUnjoined = true
T.Step("band", func(T miniprofiler.Timer) {
var d, p opentsdb.Duration
d, err = opentsdb.ParseDuration(duration)
if err != nil {
return
}
p, err = opentsdb.ParseDuration(period)
if err != nil {
return
}
if num < 1 || num > 100 {
err = fmt.Errorf("num out of bounds")
}
var q *opentsdb.Query
q, err = opentsdb.ParseQuery(query, e.tsdbContext.Version())
if err != nil {
return
}
if !e.tsdbContext.Version().FilterSupport() {
if err = e.Search.Expand(q); err != nil {
return
}
}
req := opentsdb.Request{
Queries: []*opentsdb.Query{q},
}
now := e.now
req.End = now.Unix()
req.Start = now.Add(time.Duration(-d)).Unix()
if err = req.SetTime(e.now); err != nil {
return
}
for i := 0; i < int(num); i++ {
now = now.Add(time.Duration(-p))
req.End = now.Unix()
req.Start = now.Add(time.Duration(-d)).Unix()
var s opentsdb.ResponseSet
s, err = timeTSDBRequest(e, T, &req)
if err != nil {
return
}
for _, res := range s {
if e.squelched(res.Tags) {
continue
}
//offset := e.now.Sub(now.Add(time.Duration(p-d)))
offset := e.now.Sub(now)
if err = rfunc(r, res, offset); err != nil {
return
}
}
}
})
return
}
func (e *State) walkFunc(node *parse.FuncNode, T miniprofiler.Timer) *Results {
var res *Results
T.Step("func: "+node.Name, func(T miniprofiler.Timer) {
var in []reflect.Value
for i, a := range node.Args {
var v interface{}
switch t := a.(type) {
case *parse.StringNode:
v = t.Text
case *parse.NumberNode:
v = t.Float64
case *parse.FuncNode:
v = extract(e.walkFunc(t, T))
case *parse.UnaryNode:
v = extract(e.walkUnary(t, T))
case *parse.BinaryNode:
v = extract(e.walkBinary(t, T))
case *parse.ExprNode:
v = e.walkExpr(t, T)
default:
panic(fmt.Errorf("expr: unknown func arg type"))
}
var argType models.FuncType
if i >= len(node.F.Args) {
if !node.F.VArgs {
panic("expr: shouldn't be here, more args then expected and not variable argument type func")
}
argType = node.F.Args[node.F.VArgsPos]
} else {
argType = node.F.Args[i]
}
if f, ok := v.(float64); ok && argType == models.TypeNumberSet {
v = fromScalar(f)
}
in = append(in, reflect.ValueOf(v))
}
f := reflect.ValueOf(node.F.F)
fr := f.Call(append([]reflect.Value{reflect.ValueOf(e), reflect.ValueOf(T)}, in...))
res = fr[0].Interface().(*Results)
if len(fr) > 1 && !fr[1].IsNil() {
err := fr[1].Interface().(error)
if err != nil {
panic(err)
}
}
if node.Return() == models.TypeNumberSet {
for _, r := range res.Results {
e.AddComputation(r, node.String(), r.Value.(Number))
}
}
})
return res
}
func (e *Expr) ExecuteState(s *State, T miniprofiler.Timer) (r *Results, queries []opentsdb.Request, err error) {
defer errRecover(&err)
if T == nil {
T = new(miniprofiler.Profile)
} else {
s.enableComputations = true
}
T.Step("expr execute", func(T miniprofiler.Timer) {
r = s.walk(e.Tree.Root, T)
})
queries = s.tsdbQueries
return
}
func Index(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) {
db, _ := sql.Open("sqlite3", ":memory:")
db.ExecTimer(t, "create table x(a, b, c)")
db.ExecTimer(t, "insert into x (1, 2, 4), (3, 5, 6)")
db.QueryTimer(t, "select * from x")
t.Step("redissss", func(t miniprofiler.Timer) {
conn, _ := redis.Dial("tcp", ":6379")
defer conn.Close()
conn.DoTimer(t, "set", "tes t", "value")
conn.SendTimer(t, "get", "test t")
})
fmt.Fprintf(w, `<html><body>%v</body></html>`, t.Includes())
}
func (s *Schedule) MarshalGroups(T miniprofiler.Timer, filter string) (*StateGroups, error) {
var silenced SilenceTester
T.Step("Silenced", func(miniprofiler.Timer) {
silenced = s.Silenced()
})
var groups map[StateTuple]States
var err error
status := make(States)
t := StateGroups{
TimeAndDate: s.SystemConf.GetTimeAndDate(),
}
t.FailingAlerts, t.UnclosedErrors = s.getErrorCounts()
T.Step("Setup", func(miniprofiler.Timer) {
status2, err2 := s.GetOpenStates()
if err2 != nil {
err = err2
return
}
var parsedExpr *boolq.Tree
parsedExpr, err2 = boolq.Parse(filter)
if err2 != nil {
err = err2
return
}
for k, v := range status2 {
a := s.RuleConf.GetAlert(k.Name())
if a == nil {
slog.Errorf("unknown alert %s. Force closing.", k.Name())
if err2 = s.ActionByAlertKey("bosun", "closing because alert doesn't exist.", models.ActionForceClose, k); err2 != nil {
slog.Error(err2)
}
continue
}
is, err2 := MakeIncidentSummary(s.RuleConf, silenced, v)
if err2 != nil {
err = err2
return
}
match := false
match, err2 = boolq.AskParsedExpr(parsedExpr, is)
if err2 != nil {
err = err2
return
}
if match {
status[k] = v
}
}
})
if err != nil {
return nil, err
}
T.Step("GroupStates", func(T miniprofiler.Timer) {
groups = status.GroupStates(silenced)
})
T.Step("groups", func(T miniprofiler.Timer) {
for tuple, states := range groups {
var grouped []*StateGroup
switch tuple.Status {
case models.StWarning, models.StCritical, models.StUnknown:
var sets map[string]models.AlertKeys
T.Step(fmt.Sprintf("GroupSets (%d): %v", len(states), tuple), func(T miniprofiler.Timer) {
sets = states.GroupSets(s.SystemConf.GetMinGroupSize())
})
for name, group := range sets {
g := StateGroup{
Active: tuple.Active,
Status: tuple.Status,
CurrentStatus: tuple.CurrentStatus,
Silenced: tuple.Silenced,
Subject: fmt.Sprintf("%s - %s", tuple.Status, name),
}
for _, ak := range group {
st := status[ak]
st.Body = ""
st.EmailBody = nil
st.Attachments = nil
g.Children = append(g.Children, &StateGroup{
Active: tuple.Active,
Status: tuple.Status,
Silenced: tuple.Silenced,
AlertKey: ak,
Alert: ak.Name(),
Subject: string(st.Subject),
Ago: marshalTime(st.Last().Time),
State: st,
IsError: !s.AlertSuccessful(ak.Name()),
})
}
if len(g.Children) == 1 && g.Children[0].Subject != "" {
g.Subject = g.Children[0].Subject
}
grouped = append(grouped, &g)
}
default:
continue
}
if tuple.NeedAck {
t.Groups.NeedAck = append(t.Groups.NeedAck, grouped...)
} else {
t.Groups.Acknowledged = append(t.Groups.Acknowledged, grouped...)
}
}
})
T.Step("sort", func(T miniprofiler.Timer) {
gsort := func(grp []*StateGroup) func(i, j int) bool {
return func(i, j int) bool {
a := grp[i]
b := grp[j]
if a.Active && !b.Active {
return true
} else if !a.Active && b.Active {
return false
}
if a.Status != b.Status {
return a.Status > b.Status
}
if a.AlertKey != b.AlertKey {
return a.AlertKey < b.AlertKey
}
return a.Subject < b.Subject
}
}
slice.Sort(t.Groups.NeedAck, gsort(t.Groups.NeedAck))
slice.Sort(t.Groups.Acknowledged, gsort(t.Groups.Acknowledged))
})
return &t, nil
}
func Over(e *State, T miniprofiler.Timer, query, duration, period string, num float64) (r *Results, err error) {
r = new(Results)
r.IgnoreOtherUnjoined = true
r.IgnoreUnjoined = true
T.Step("band", func(T miniprofiler.Timer) {
var d, p opentsdb.Duration
d, err = opentsdb.ParseDuration(duration)
if err != nil {
return
}
p, err = opentsdb.ParseDuration(period)
if err != nil {
return
}
if num < 1 || num > 100 {
err = fmt.Errorf("num out of bounds")
}
var q *opentsdb.Query
q, err = opentsdb.ParseQuery(query, e.tsdbContext.Version())
if err != nil {
return
}
if !e.tsdbContext.Version().FilterSupport() {
if err = e.Search.Expand(q); err != nil {
return
}
}
req := opentsdb.Request{
Queries: []*opentsdb.Query{q},
}
now := e.now
req.End = now.Unix()
req.Start = now.Add(time.Duration(-d)).Unix()
for i := 0; i < int(num); i++ {
var s opentsdb.ResponseSet
s, err = timeTSDBRequest(e, T, &req)
if err != nil {
return
}
offset := e.now.Sub(now)
for _, res := range s {
if e.squelched(res.Tags) {
continue
}
values := make(Series)
a := &Result{Group: res.Tags.Merge(opentsdb.TagSet{"shift": offset.String()})}
for k, v := range res.DPS {
i, err := strconv.ParseInt(k, 10, 64)
if err != nil {
return
}
values[time.Unix(i, 0).Add(offset).UTC()] = float64(v)
}
a.Value = values
r.Results = append(r.Results, a)
}
now = now.Add(time.Duration(-p))
req.End = now.Unix()
req.Start = now.Add(time.Duration(-d)).Unix()
}
})
return
}
func GraphiteBand(e *State, T miniprofiler.Timer, query, duration, period, format string, num float64) (r *Results, err error) {
r = new(Results)
r.IgnoreOtherUnjoined = true
r.IgnoreUnjoined = true
T.Step("graphiteBand", func(T miniprofiler.Timer) {
var d, p opentsdb.Duration
d, err = opentsdb.ParseDuration(duration)
if err != nil {
return
}
p, err = opentsdb.ParseDuration(period)
if err != nil {
return
}
if num < 1 || num > 100 {
err = fmt.Errorf("expr: Band: num out of bounds")
}
req := &graphite.Request{
Targets: []string{query},
}
now := e.now
req.End = &now
st := e.now.Add(-time.Duration(d))
req.Start = &st
for i := 0; i < int(num); i++ {
now = now.Add(time.Duration(-p))
req.End = &now
st := now.Add(time.Duration(-d))
req.Start = &st
var s graphite.Response
s, err = timeGraphiteRequest(e, T, req)
if err != nil {
return
}
formatTags := strings.Split(format, ".")
var results []*Result
results, err = parseGraphiteResponse(req, &s, formatTags)
if err != nil {
return
}
if i == 0 {
r.Results = results
} else {
// different graphite requests might return series with different id's.
// i.e. a different set of tagsets. merge the data of corresponding tagsets
for _, result := range results {
updateKey := -1
for j, existing := range r.Results {
if result.Group.Equal(existing.Group) {
updateKey = j
break
}
}
if updateKey == -1 {
// result tagset is new
r.Results = append(r.Results, result)
updateKey = len(r.Results) - 1
}
for k, v := range result.Value.(Series) {
r.Results[updateKey].Value.(Series)[k] = v
}
}
}
}
})
if err != nil {
return nil, fmt.Errorf("graphiteBand: %v", err)
}
return
}
func Rule(t miniprofiler.Timer, w http.ResponseWriter, r *http.Request) (interface{}, error) {
var from, to time.Time
var err error
if f := r.FormValue("from"); len(f) > 0 {
from, err = time.Parse(tsdbFormatSecs, f)
if err != nil {
return nil, err
}
}
if f := r.FormValue("to"); len(f) > 0 {
to, err = time.Parse(tsdbFormatSecs, f)
if err != nil {
return nil, err
}
}
intervals := 1
if i := r.FormValue("intervals"); len(i) > 0 {
intervals, err = strconv.Atoi(r.FormValue("intervals"))
if err != nil {
return nil, err
}
if intervals < 1 {
return nil, fmt.Errorf("must be > 0 intervals")
}
}
if fz, tz := from.IsZero(), to.IsZero(); fz && tz {
from = time.Now()
} else if fz && !tz {
return nil, fmt.Errorf("cannot specify to without from")
} else if !fz && tz && intervals > 1 {
return nil, fmt.Errorf("cannot specify intervals without from and to")
}
c, a, hash, err := buildConfig(r)
if err != nil {
return nil, err
}
ch := make(chan int)
errch := make(chan error, intervals)
resch := make(chan *ruleResult, intervals)
var wg sync.WaitGroup
diff := -from.Sub(to)
if intervals > 1 {
diff /= time.Duration(intervals - 1)
}
worker := func() {
wg.Add(1)
for interval := range ch {
t.Step(fmt.Sprintf("interval %v", interval), func(t miniprofiler.Timer) {
now := from.Add(diff * time.Duration(interval))
res, err := procRule(t, c, a, now, interval != 0, r.FormValue("email"), r.FormValue("template_group"))
resch <- res
errch <- err
})
}
defer wg.Done()
}
for i := 0; i < 20; i++ {
go worker()
}
for i := 0; i < intervals; i++ {
ch <- i
}
close(ch)
wg.Wait()
close(errch)
close(resch)
type Result struct {
Group models.AlertKey
Result *models.Event
}
type Set struct {
Critical, Warning, Normal int
Time string
Results []*Result `json:",omitempty"`
}
type History struct {
Time, EndTime time.Time
Status string
}
type Histories struct {
History []*History
}
ret := struct {
Errors []string `json:",omitempty"`
Warnings []string `json:",omitempty"`
Sets []*Set
AlertHistory map[models.AlertKey]*Histories
Body string `json:",omitempty"`
Subject string `json:",omitempty"`
Data interface{} `json:",omitempty"`
Hash string
}{
AlertHistory: make(map[models.AlertKey]*Histories),
Hash: hash,
}
for err := range errch {
if err == nil {
continue
}
ret.Errors = append(ret.Errors, err.Error())
}
for res := range resch {
if res == nil {
continue
}
set := Set{
Critical: len(res.Criticals),
Warning: len(res.Warnings),
Normal: len(res.Normals),
Time: res.Time.Format(tsdbFormatSecs),
}
if res.Data != nil {
ret.Body = res.Body
ret.Subject = res.Subject
ret.Data = res.Data
for k, v := range res.Result {
set.Results = append(set.Results, &Result{
Group: k,
Result: v,
})
}
slice.Sort(set.Results, func(i, j int) bool {
a := set.Results[i]
b := set.Results[j]
if a.Result.Status != b.Result.Status {
return a.Result.Status > b.Result.Status
}
return a.Group < b.Group
})
}
for k, v := range res.Result {
if ret.AlertHistory[k] == nil {
ret.AlertHistory[k] = new(Histories)
}
h := ret.AlertHistory[k]
h.History = append(h.History, &History{
Time: v.Time,
Status: v.Status.String(),
})
}
ret.Sets = append(ret.Sets, &set)
ret.Warnings = append(ret.Warnings, res.Warning...)
}
slice.Sort(ret.Sets, func(i, j int) bool {
return ret.Sets[i].Time < ret.Sets[j].Time
})
for _, histories := range ret.AlertHistory {
hist := histories.History
slice.Sort(hist, func(i, j int) bool {
return hist[i].Time.Before(hist[j].Time)
})
for i := 1; i < len(hist); i++ {
if i < len(hist)-1 && hist[i].Status == hist[i-1].Status {
hist = append(hist[:i], hist[i+1:]...)
i--
}
}
for i, h := range hist[:len(hist)-1] {
h.EndTime = hist[i+1].Time
}
histories.History = hist[:len(hist)-1]
}
return &ret, nil
}
func (e *State) walkBinary(node *parse.BinaryNode, T miniprofiler.Timer) *Results {
ar := e.walk(node.Args[0], T)
br := e.walk(node.Args[1], T)
res := Results{
IgnoreUnjoined: ar.IgnoreUnjoined || br.IgnoreUnjoined,
IgnoreOtherUnjoined: ar.IgnoreOtherUnjoined || br.IgnoreOtherUnjoined,
}
T.Step("walkBinary: "+node.OpStr, func(T miniprofiler.Timer) {
u := e.union(ar, br, node.String())
for _, v := range u {
var value Value
r := &Result{
Group: v.Group,
Computations: v.Computations,
}
switch at := v.A.(type) {
case Scalar:
switch bt := v.B.(type) {
case Scalar:
n := Scalar(operate(node.OpStr, float64(at), float64(bt)))
e.AddComputation(r, node.String(), Number(n))
value = n
case Number:
n := Number(operate(node.OpStr, float64(at), float64(bt)))
e.AddComputation(r, node.String(), n)
value = n
case Series:
s := make(Series)
for k, v := range bt {
s[k] = operate(node.OpStr, float64(at), float64(v))
}
value = s
default:
panic(ErrUnknownOp)
}
case Number:
switch bt := v.B.(type) {
case Scalar:
n := Number(operate(node.OpStr, float64(at), float64(bt)))
e.AddComputation(r, node.String(), Number(n))
value = n
case Number:
n := Number(operate(node.OpStr, float64(at), float64(bt)))
e.AddComputation(r, node.String(), n)
value = n
case Series:
s := make(Series)
for k, v := range bt {
s[k] = operate(node.OpStr, float64(at), float64(v))
}
value = s
default:
panic(ErrUnknownOp)
}
case Series:
switch bt := v.B.(type) {
case Number, Scalar:
bv := reflect.ValueOf(bt).Float()
s := make(Series)
for k, v := range at {
s[k] = operate(node.OpStr, float64(v), bv)
}
value = s
case Series:
s := make(Series)
for k, av := range at {
if bv, ok := bt[k]; ok {
s[k] = operate(node.OpStr, av, bv)
}
}
value = s
default:
panic(ErrUnknownOp)
}
default:
panic(ErrUnknownOp)
}
r.Value = value
res.Results = append(res.Results, r)
}
})
return &res
}
func bandRangeTSDB(e *State, T miniprofiler.Timer, query, rangeStart, rangeEnd, period string, num float64, rfunc func(*Results, *opentsdb.Response) error) (r *Results, err error) {
r = new(Results)
r.IgnoreOtherUnjoined = true
r.IgnoreUnjoined = true
T.Step("bandRange", func(T miniprofiler.Timer) {
var from, to, p opentsdb.Duration
from, err = opentsdb.ParseDuration(rangeStart)
if err != nil {
return
}
to, err = opentsdb.ParseDuration(rangeEnd)
if err != nil {
return
}
p, err = opentsdb.ParseDuration(period)
if err != nil {
return
}
if num < 1 || num > 100 {
err = fmt.Errorf("num out of bounds")
}
var q *opentsdb.Query
q, err = opentsdb.ParseQuery(query, e.tsdbContext.Version())
if err != nil {
return
}
if err = e.Search.Expand(q); err != nil {
return
}
req := opentsdb.Request{
Queries: []*opentsdb.Query{q},
}
now := e.now
req.End = now.Unix()
st := now.Add(time.Duration(from)).Unix()
req.Start = st
if err = req.SetTime(e.now); err != nil {
return
}
for i := 0; i < int(num); i++ {
now = now.Add(time.Duration(-p))
end := now.Add(time.Duration(-to)).Unix()
req.End = &end
st := now.Add(time.Duration(-from)).Unix()
req.Start = &st
var s opentsdb.ResponseSet
s, err = timeTSDBRequest(e, T, &req)
if err != nil {
return
}
for _, res := range s {
if e.squelched(res.Tags) {
continue
}
if err = rfunc(r, res); err != nil {
return
}
}
}
})
return
}