func (s *Schedule) executeTemplates(state *State, event *Event, a *conf.Alert, r *RunHistory) {
state.Subject = ""
state.Body = ""
state.EmailBody = nil
state.EmailSubject = nil
state.Attachments = nil
if event.Status != StUnknown {
metric := "template.render"
//Render subject
endTiming := collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "subject"})
subject, serr := s.ExecuteSubject(r, a, state, false)
if serr != nil {
slog.Infof("%s: %v", state.AlertKey(), serr)
}
endTiming()
//Render body
endTiming = collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "body"})
body, _, berr := s.ExecuteBody(r, a, state, false)
if berr != nil {
slog.Infof("%s: %v", state.AlertKey(), berr)
}
endTiming()
//Render email body
endTiming = collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "emailbody"})
emailbody, attachments, merr := s.ExecuteBody(r, a, state, true)
if merr != nil {
slog.Infof("%s: %v", state.AlertKey(), merr)
}
endTiming()
//Render email subject
endTiming = collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "emailsubject"})
emailsubject, eserr := s.ExecuteSubject(r, a, state, true)
endTiming()
if serr != nil || berr != nil || merr != nil || eserr != nil {
var err error
endTiming = collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "bad"})
subject, body, err = s.ExecuteBadTemplate(serr, berr, r, a, state)
endTiming()
if err != nil {
subject = []byte(fmt.Sprintf("unable to create template error notification: %v", err))
}
emailbody = body
attachments = nil
}
state.Subject = string(subject)
state.Body = string(body)
state.EmailBody = emailbody
state.EmailSubject = emailsubject
state.Attachments = attachments
}
}
func (d *dataAccess) GetActiveSilences() ([]*models.Silence, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetActiveSilences"})()
conn := d.GetConnection()
defer conn.Close()
now := time.Now().UTC()
vals, err := redis.Strings(conn.Do("ZRANGEBYSCORE", silenceIdx, now.Unix(), "+inf"))
if err != nil {
return nil, err
}
if len(vals) == 0 {
return nil, nil
}
silences, err := getSilences(vals, conn)
if err != nil {
return nil, err
}
filtered := make([]*models.Silence, 0, len(silences))
for _, s := range silences {
if s.Start.After(now) {
continue
}
filtered = append(filtered, s)
}
return filtered, nil
}
func (d *dataAccess) GetIncidentsStartingInRange(start, end time.Time) ([]*models.Incident, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetIncidentsStartingInRange"})()
conn := d.GetConnection()
defer conn.Close()
ids, err := redis.Ints(conn.Do("ZRANGEBYSCORE", "incidentsByStart", start.UTC().Unix(), end.UTC().Unix()))
if err != nil {
return nil, err
}
args := make([]interface{}, len(ids)+1)
args[0] = "incidents"
for i := range ids {
args[i+1] = ids[i]
}
jsons, err := redis.Strings(conn.Do("HMGET", args...))
if err != nil {
return nil, err
}
incidents := make([]*models.Incident, len(jsons))
for i := range jsons {
inc := &models.Incident{}
if err = json.Unmarshal([]byte(jsons[i]), inc); err != nil {
return nil, err
}
incidents[i] = inc
}
return incidents, nil
}
//Things could forseeably get a bit inconsistent if concurrent changes happen in just the wrong way.
//Clear all should do a more thourogh cleanup to fully reset things.
func (d *dataAccess) ClearAll() error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "ClearAll"})()
conn := d.GetConnection()
defer conn.Close()
alerts, err := redis.Strings(conn.Do("SMEMBERS", alertsWithErrors))
if err != nil {
return err
}
for _, a := range alerts {
if _, err := conn.Do(d.LCLEAR(), errorListKey(a)); err != nil {
return err
}
}
if _, err := conn.Do(d.SCLEAR(), alertsWithErrors); err != nil {
return err
}
if _, err := conn.Do(d.SCLEAR(), failingAlerts); err != nil {
return err
}
if _, err = conn.Do(d.LCLEAR(), errorEvents); err != nil {
return err
}
return nil
}
func (d *dataAccess) MarkAlertSuccess(name string) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "MarkAlertSuccess"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("SREM", failingAlerts, name)
return err
}
func (d *dataAccess) ClearAlert(name string) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "ClearAlert"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("SREM", alertsWithErrors, name)
if err != nil {
return err
}
_, err = conn.Do("SREM", failingAlerts, name)
if err != nil {
return err
}
_, err = conn.Do(d.LCLEAR(), errorListKey(name))
if err != nil {
return err
}
cmd, args := d.LMCLEAR(errorEvents, name)
_, err = conn.Do(cmd, args...)
if err != nil {
return err
}
return nil
}
func (d *dataAccess) GetFullErrorHistory() (map[string][]*models.AlertError, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetFullErrorHistory"})()
conn := d.GetConnection()
defer conn.Close()
alerts, err := redis.Strings(conn.Do("SMEMBERS", alertsWithErrors))
if err != nil {
return nil, err
}
results := make(map[string][]*models.AlertError, len(alerts))
for _, a := range alerts {
rows, err := redis.Strings(conn.Do("LRANGE", errorListKey(a), 0, -1))
if err != nil {
return nil, err
}
list := make([]*models.AlertError, len(rows))
for i, row := range rows {
ae := &models.AlertError{}
err = json.Unmarshal([]byte(row), ae)
if err != nil {
return nil, err
}
list[i] = ae
}
results[a] = list
}
return results, nil
}
//This function not exposed on any public interface. See cmd/bosun/database/test/util/purge_search_data.go for usage.
func (d *dataAccess) PurgeSearchData(metric string, noop bool) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "PurgeSearchData"})()
conn := d.GetConnection()
defer conn.Close()
tagKeys, err := d.GetTagKeysForMetric(metric)
if err != nil {
return err
}
fmt.Println("HDEL", searchAllMetricsKey)
if !noop {
_, err = conn.Do("HDEL", searchAllMetricsKey, metric)
if err != nil {
return err
}
}
hashesToDelete := []string{
searchMetricTagSetKey(metric),
searchTagkKey(metric),
}
for tagk := range tagKeys {
hashesToDelete = append(hashesToDelete, searchTagvKey(metric, tagk))
}
cmd := d.HCLEAR()
for _, hash := range hashesToDelete {
fmt.Println(cmd, hash)
if !noop {
_, err = conn.Do(cmd, hash)
if err != nil {
return err
}
}
}
return nil
}
func (d *dataAccess) GetTagValues(metric, tagK string) (map[string]int64, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetTagValues"})()
conn := d.GetConnection()
defer conn.Close()
return stringInt64Map(conn.Do("HGETALL", searchTagvKey(metric, tagK)))
}
func (d *dataAccess) GetLatestIncident(ak models.AlertKey) (*models.IncidentState, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetLatestIncident"})()
conn := d.GetConnection()
defer conn.Close()
return d.getLatestIncident(ak, conn)
}
func newPool(server, password string, database int, isRedis bool) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", server, redis.DialDatabase(database))
if err != nil {
return nil, err
}
if password != "" {
if _, err := c.Do("AUTH", password); err != nil {
c.Close()
return nil, err
}
}
if isRedis {
if _, err := c.Do("CLIENT", "SETNAME", "bosun"); err != nil {
c.Close()
return nil, err
}
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "Ping"})()
_, err := c.Do("PING")
return err
},
}
}
func (d *dataAccess) GetMetricsForTag(tagK, tagV string) (map[string]int64, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetMetricsForTag"})()
conn := d.GetConnection()
defer conn.Close()
return stringInt64Map(conn.Do("HGETALL", searchMetricKey(tagK, tagV)))
}
func (d *dataAccess) Search_GetAllMetrics() (map[string]int64, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetAllMetrics"})()
conn := d.GetConnection()
defer conn.Close()
return stringInt64Map(conn.Do("HGETALL", searchAllMetricsKey))
}
func (d *dataAccess) Search_GetTagKeysForMetric(metric string) (map[string]int64, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetTagKeysForMetric"})()
conn := d.GetConnection()
defer conn.Close()
return stringInt64Map(conn.Do("HGETALL", searchTagkKey(metric)))
}
func (d *dataAccess) Get() redis.Conn {
closer := collect.StartTimer("redis", opentsdb.TagSet{"op": myCallerName()})
return &connWrapper{
Conn: d.pool.Get(),
closer: closer,
}
}
func (d *dataAccess) GetUnknownAndUnevalAlertKeys(alert string) ([]models.AlertKey, []models.AlertKey, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetUnknownAndUnevalAlertKeys"})()
conn := d.GetConnection()
defer conn.Close()
unknownS, err := redis.Strings(conn.Do("SMEMBERS", statesUnknownKey(alert)))
if err != nil {
return nil, nil, slog.Wrap(err)
}
unknown := make([]models.AlertKey, len(unknownS))
for i, u := range unknownS {
unknown[i] = models.AlertKey(u)
}
unEvals, err := redis.Strings(conn.Do("SMEMBERS", statesUnevalKey(alert)))
if err != nil {
return nil, nil, slog.Wrap(err)
}
unevals := make([]models.AlertKey, len(unEvals))
for i, u := range unEvals {
unevals[i] = models.AlertKey(u)
}
return unknown, unevals, nil
}
func (d *dataAccess) AddMetricTagSet(metric, tagSet string, time int64) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddMetricTagSet"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("HSET", searchMetricTagSetKey(metric), tagSet, time)
return err
}
func (d *dataAccess) AddTagKeyForMetric(metric, tagK string, time int64) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddTagKeyForMetric"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("HSET", searchTagkKey(metric), tagK, time)
return slog.Wrap(err)
}
func (d *dataAccess) TouchAlertKey(ak models.AlertKey, t time.Time) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "TouchAlertKey"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("ZADD", statesLastTouchedKey(ak.Name()), t.UTC().Unix(), string(ak))
return slog.Wrap(err)
}
func (d *dataAccess) AddMetricForTag(tagK, tagV, metric string, time int64) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddMetricForTag"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("HSET", searchMetricKey(tagK, tagV), metric, time)
return slog.Wrap(err)
}
func (d *dataAccess) SetMaxId(id uint64) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "SetMaxId"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("SET", "maxIncidentId", id)
return err
}
func (d *dataAccess) ClearNotificationsBefore(t time.Time) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "ClearNotificationsBefore"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("ZREMRANGEBYSCORE", pendingNotificationsKey, 0, t.UTC().Unix())
return slog.Wrap(err)
}
func (d *dataAccess) Search_AddMetric(metric string, time int64) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddMetric"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("HSET", searchAllMetricsKey, metric, time)
return err
}
func (d *dataAccess) Search_AddTagValue(metric, tagK, tagV string, time int64) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddTagValue"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("HSET", searchTagvKey(metric, tagK), tagV, time)
return err
}
func (d *dataAccess) SaveTempConfig(text string) (string, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "SaveTempConfig"})()
conn := d.GetConnection()
defer conn.Close()
sig := md5.Sum([]byte(text))
b64 := base64.StdEncoding.EncodeToString(sig[0:8])
_, err := conn.Do("SET", "tempConfig:"+b64, text, "EX", configLifetime)
return b64, err
}
func (d *dataAccess) PutMetricMetadata(metric string, field string, value string) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "PutMetricMeta"})()
if field != "desc" && field != "unit" && field != "rate" {
return fmt.Errorf("Unknown metric metadata field: %s", field)
}
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("HMSET", metricMetaKey(metric), field, value, "lastTouched", time.Now().UTC().Unix())
return err
}
func (d *dataAccess) GetAllIncidents(ak models.AlertKey) ([]*models.IncidentState, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetAllIncidents"})()
conn := d.GetConnection()
defer conn.Close()
ids, err := int64s(conn.Do("LRANGE", incidentsForAlertKeyKey(ak), 0, -1))
if err != nil {
return nil, slog.Wrap(err)
}
return d.incidentMultiGet(conn, ids)
}
func (d *dataAccess) InsertNotification(ak models.AlertKey, notification string, dueAt time.Time) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "InsertNotification"})()
conn := d.GetConnection()
defer conn.Close()
_, err := conn.Do("ZADD", pendingNotificationsKey, dueAt.UTC().Unix(), fmt.Sprintf("%s:%s", ak, notification))
if err != nil {
return slog.Wrap(err)
}
_, err = conn.Do("SADD", notsByAlertKeyKey(ak), notification)
return slog.Wrap(err)
}
func (d *dataAccess) GetAllOpenIncidents() ([]*models.IncidentState, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetAllOpenIncidents"})()
conn := d.GetConnection()
defer conn.Close()
// get open ids
ids, err := int64s(conn.Do("HVALS", statesOpenIncidentsKey))
if err != nil {
return nil, slog.Wrap(err)
}
return d.incidentMultiGet(conn, ids)
}
func (d *dataAccess) SetUnevaluated(ak models.AlertKey, uneval bool) error {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "SetUnevaluated"})()
conn := d.GetConnection()
defer conn.Close()
op := "SREM"
if uneval {
op = "SADD"
}
_, err := conn.Do(op, statesUnevalKey(ak.Name()), ak)
return slog.Wrap(err)
}