Esempio n. 1
0
func (s *Schedule) executeTemplates(state *State, event *Event, a *conf.Alert, r *RunHistory) {
	state.Subject = ""
	state.Body = ""
	state.EmailBody = nil
	state.EmailSubject = nil
	state.Attachments = nil
	if event.Status != StUnknown {
		metric := "template.render"
		//Render subject
		endTiming := collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "subject"})
		subject, serr := s.ExecuteSubject(r, a, state, false)
		if serr != nil {
			slog.Infof("%s: %v", state.AlertKey(), serr)
		}
		endTiming()
		//Render body
		endTiming = collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "body"})
		body, _, berr := s.ExecuteBody(r, a, state, false)
		if berr != nil {
			slog.Infof("%s: %v", state.AlertKey(), berr)
		}
		endTiming()
		//Render email body
		endTiming = collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "emailbody"})
		emailbody, attachments, merr := s.ExecuteBody(r, a, state, true)
		if merr != nil {
			slog.Infof("%s: %v", state.AlertKey(), merr)
		}
		endTiming()
		//Render email subject
		endTiming = collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "emailsubject"})
		emailsubject, eserr := s.ExecuteSubject(r, a, state, true)
		endTiming()
		if serr != nil || berr != nil || merr != nil || eserr != nil {
			var err error

			endTiming = collect.StartTimer(metric, opentsdb.TagSet{"alert": a.Name, "type": "bad"})
			subject, body, err = s.ExecuteBadTemplate(serr, berr, r, a, state)
			endTiming()

			if err != nil {
				subject = []byte(fmt.Sprintf("unable to create template error notification: %v", err))
			}
			emailbody = body
			attachments = nil
		}
		state.Subject = string(subject)
		state.Body = string(body)
		state.EmailBody = emailbody
		state.EmailSubject = emailsubject
		state.Attachments = attachments
	}
}
Esempio n. 2
0
func (d *dataAccess) GetActiveSilences() ([]*models.Silence, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetActiveSilences"})()
	conn := d.GetConnection()
	defer conn.Close()

	now := time.Now().UTC()
	vals, err := redis.Strings(conn.Do("ZRANGEBYSCORE", silenceIdx, now.Unix(), "+inf"))
	if err != nil {
		return nil, err
	}
	if len(vals) == 0 {
		return nil, nil
	}
	silences, err := getSilences(vals, conn)
	if err != nil {
		return nil, err
	}
	filtered := make([]*models.Silence, 0, len(silences))
	for _, s := range silences {
		if s.Start.After(now) {
			continue
		}
		filtered = append(filtered, s)
	}
	return filtered, nil
}
Esempio n. 3
0
func (d *dataAccess) GetIncidentsStartingInRange(start, end time.Time) ([]*models.Incident, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetIncidentsStartingInRange"})()
	conn := d.GetConnection()
	defer conn.Close()

	ids, err := redis.Ints(conn.Do("ZRANGEBYSCORE", "incidentsByStart", start.UTC().Unix(), end.UTC().Unix()))
	if err != nil {
		return nil, err
	}
	args := make([]interface{}, len(ids)+1)
	args[0] = "incidents"
	for i := range ids {
		args[i+1] = ids[i]
	}
	jsons, err := redis.Strings(conn.Do("HMGET", args...))
	if err != nil {
		return nil, err
	}
	incidents := make([]*models.Incident, len(jsons))
	for i := range jsons {
		inc := &models.Incident{}
		if err = json.Unmarshal([]byte(jsons[i]), inc); err != nil {
			return nil, err
		}
		incidents[i] = inc
	}
	return incidents, nil
}
Esempio n. 4
0
//Things could forseeably get a bit inconsistent if concurrent changes happen in just the wrong way.
//Clear all should do a more thourogh cleanup to fully reset things.
func (d *dataAccess) ClearAll() error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "ClearAll"})()
	conn := d.GetConnection()
	defer conn.Close()

	alerts, err := redis.Strings(conn.Do("SMEMBERS", alertsWithErrors))
	if err != nil {
		return err
	}
	for _, a := range alerts {
		if _, err := conn.Do(d.LCLEAR(), errorListKey(a)); err != nil {
			return err
		}
	}
	if _, err := conn.Do(d.SCLEAR(), alertsWithErrors); err != nil {
		return err
	}
	if _, err := conn.Do(d.SCLEAR(), failingAlerts); err != nil {
		return err
	}
	if _, err = conn.Do(d.LCLEAR(), errorEvents); err != nil {
		return err
	}

	return nil
}
Esempio n. 5
0
func (d *dataAccess) MarkAlertSuccess(name string) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "MarkAlertSuccess"})()
	conn := d.GetConnection()
	defer conn.Close()
	_, err := conn.Do("SREM", failingAlerts, name)
	return err
}
Esempio n. 6
0
func (d *dataAccess) ClearAlert(name string) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "ClearAlert"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("SREM", alertsWithErrors, name)
	if err != nil {
		return err
	}
	_, err = conn.Do("SREM", failingAlerts, name)
	if err != nil {
		return err
	}
	_, err = conn.Do(d.LCLEAR(), errorListKey(name))
	if err != nil {
		return err
	}
	cmd, args := d.LMCLEAR(errorEvents, name)
	_, err = conn.Do(cmd, args...)
	if err != nil {
		return err
	}

	return nil
}
Esempio n. 7
0
func (d *dataAccess) GetFullErrorHistory() (map[string][]*models.AlertError, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetFullErrorHistory"})()
	conn := d.GetConnection()
	defer conn.Close()

	alerts, err := redis.Strings(conn.Do("SMEMBERS", alertsWithErrors))
	if err != nil {
		return nil, err
	}
	results := make(map[string][]*models.AlertError, len(alerts))
	for _, a := range alerts {
		rows, err := redis.Strings(conn.Do("LRANGE", errorListKey(a), 0, -1))
		if err != nil {
			return nil, err
		}
		list := make([]*models.AlertError, len(rows))
		for i, row := range rows {
			ae := &models.AlertError{}
			err = json.Unmarshal([]byte(row), ae)
			if err != nil {
				return nil, err
			}
			list[i] = ae
		}
		results[a] = list
	}
	return results, nil
}
Esempio n. 8
0
//This function not exposed on any public interface. See cmd/bosun/database/test/util/purge_search_data.go for usage.
func (d *dataAccess) PurgeSearchData(metric string, noop bool) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "PurgeSearchData"})()
	conn := d.GetConnection()
	defer conn.Close()

	tagKeys, err := d.GetTagKeysForMetric(metric)
	if err != nil {
		return err
	}
	fmt.Println("HDEL", searchAllMetricsKey)
	if !noop {
		_, err = conn.Do("HDEL", searchAllMetricsKey, metric)
		if err != nil {
			return err
		}
	}
	hashesToDelete := []string{
		searchMetricTagSetKey(metric),
		searchTagkKey(metric),
	}
	for tagk := range tagKeys {
		hashesToDelete = append(hashesToDelete, searchTagvKey(metric, tagk))
	}
	cmd := d.HCLEAR()
	for _, hash := range hashesToDelete {
		fmt.Println(cmd, hash)
		if !noop {
			_, err = conn.Do(cmd, hash)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
Esempio n. 9
0
func (d *dataAccess) GetTagValues(metric, tagK string) (map[string]int64, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetTagValues"})()
	conn := d.GetConnection()
	defer conn.Close()

	return stringInt64Map(conn.Do("HGETALL", searchTagvKey(metric, tagK)))
}
Esempio n. 10
0
func (d *dataAccess) GetLatestIncident(ak models.AlertKey) (*models.IncidentState, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetLatestIncident"})()
	conn := d.GetConnection()
	defer conn.Close()

	return d.getLatestIncident(ak, conn)
}
Esempio n. 11
0
func newPool(server, password string, database int, isRedis bool) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     3,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", server, redis.DialDatabase(database))
			if err != nil {
				return nil, err
			}
			if password != "" {
				if _, err := c.Do("AUTH", password); err != nil {
					c.Close()
					return nil, err
				}
			}
			if isRedis {
				if _, err := c.Do("CLIENT", "SETNAME", "bosun"); err != nil {
					c.Close()
					return nil, err
				}
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			defer collect.StartTimer("redis", opentsdb.TagSet{"op": "Ping"})()
			_, err := c.Do("PING")
			return err
		},
	}
}
Esempio n. 12
0
func (d *dataAccess) GetMetricsForTag(tagK, tagV string) (map[string]int64, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetMetricsForTag"})()
	conn := d.GetConnection()
	defer conn.Close()

	return stringInt64Map(conn.Do("HGETALL", searchMetricKey(tagK, tagV)))
}
Esempio n. 13
0
func (d *dataAccess) Search_GetAllMetrics() (map[string]int64, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetAllMetrics"})()
	conn := d.GetConnection()
	defer conn.Close()

	return stringInt64Map(conn.Do("HGETALL", searchAllMetricsKey))
}
Esempio n. 14
0
func (d *dataAccess) Search_GetTagKeysForMetric(metric string) (map[string]int64, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetTagKeysForMetric"})()
	conn := d.GetConnection()
	defer conn.Close()

	return stringInt64Map(conn.Do("HGETALL", searchTagkKey(metric)))
}
Esempio n. 15
0
func (d *dataAccess) Get() redis.Conn {
	closer := collect.StartTimer("redis", opentsdb.TagSet{"op": myCallerName()})
	return &connWrapper{
		Conn:   d.pool.Get(),
		closer: closer,
	}
}
Esempio n. 16
0
func (d *dataAccess) GetUnknownAndUnevalAlertKeys(alert string) ([]models.AlertKey, []models.AlertKey, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetUnknownAndUnevalAlertKeys"})()
	conn := d.GetConnection()
	defer conn.Close()

	unknownS, err := redis.Strings(conn.Do("SMEMBERS", statesUnknownKey(alert)))
	if err != nil {
		return nil, nil, slog.Wrap(err)
	}
	unknown := make([]models.AlertKey, len(unknownS))
	for i, u := range unknownS {
		unknown[i] = models.AlertKey(u)
	}

	unEvals, err := redis.Strings(conn.Do("SMEMBERS", statesUnevalKey(alert)))
	if err != nil {
		return nil, nil, slog.Wrap(err)
	}
	unevals := make([]models.AlertKey, len(unEvals))
	for i, u := range unEvals {
		unevals[i] = models.AlertKey(u)
	}

	return unknown, unevals, nil
}
Esempio n. 17
0
func (d *dataAccess) AddMetricTagSet(metric, tagSet string, time int64) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddMetricTagSet"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("HSET", searchMetricTagSetKey(metric), tagSet, time)
	return err
}
Esempio n. 18
0
func (d *dataAccess) AddTagKeyForMetric(metric, tagK string, time int64) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddTagKeyForMetric"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("HSET", searchTagkKey(metric), tagK, time)
	return slog.Wrap(err)
}
Esempio n. 19
0
func (d *dataAccess) TouchAlertKey(ak models.AlertKey, t time.Time) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "TouchAlertKey"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("ZADD", statesLastTouchedKey(ak.Name()), t.UTC().Unix(), string(ak))
	return slog.Wrap(err)
}
Esempio n. 20
0
func (d *dataAccess) AddMetricForTag(tagK, tagV, metric string, time int64) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddMetricForTag"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("HSET", searchMetricKey(tagK, tagV), metric, time)
	return slog.Wrap(err)
}
Esempio n. 21
0
func (d *dataAccess) SetMaxId(id uint64) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "SetMaxId"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("SET", "maxIncidentId", id)
	return err
}
Esempio n. 22
0
func (d *dataAccess) ClearNotificationsBefore(t time.Time) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "ClearNotificationsBefore"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("ZREMRANGEBYSCORE", pendingNotificationsKey, 0, t.UTC().Unix())
	return slog.Wrap(err)
}
Esempio n. 23
0
func (d *dataAccess) Search_AddMetric(metric string, time int64) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddMetric"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("HSET", searchAllMetricsKey, metric, time)
	return err
}
Esempio n. 24
0
func (d *dataAccess) Search_AddTagValue(metric, tagK, tagV string, time int64) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "AddTagValue"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("HSET", searchTagvKey(metric, tagK), tagV, time)
	return err
}
Esempio n. 25
0
func (d *dataAccess) SaveTempConfig(text string) (string, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "SaveTempConfig"})()
	conn := d.GetConnection()
	defer conn.Close()

	sig := md5.Sum([]byte(text))
	b64 := base64.StdEncoding.EncodeToString(sig[0:8])
	_, err := conn.Do("SET", "tempConfig:"+b64, text, "EX", configLifetime)
	return b64, err
}
Esempio n. 26
0
func (d *dataAccess) PutMetricMetadata(metric string, field string, value string) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "PutMetricMeta"})()
	if field != "desc" && field != "unit" && field != "rate" {
		return fmt.Errorf("Unknown metric metadata field: %s", field)
	}
	conn := d.GetConnection()
	defer conn.Close()
	_, err := conn.Do("HMSET", metricMetaKey(metric), field, value, "lastTouched", time.Now().UTC().Unix())
	return err
}
Esempio n. 27
0
func (d *dataAccess) GetAllIncidents(ak models.AlertKey) ([]*models.IncidentState, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetAllIncidents"})()
	conn := d.GetConnection()
	defer conn.Close()

	ids, err := int64s(conn.Do("LRANGE", incidentsForAlertKeyKey(ak), 0, -1))
	if err != nil {
		return nil, slog.Wrap(err)
	}
	return d.incidentMultiGet(conn, ids)
}
Esempio n. 28
0
func (d *dataAccess) InsertNotification(ak models.AlertKey, notification string, dueAt time.Time) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "InsertNotification"})()
	conn := d.GetConnection()
	defer conn.Close()

	_, err := conn.Do("ZADD", pendingNotificationsKey, dueAt.UTC().Unix(), fmt.Sprintf("%s:%s", ak, notification))
	if err != nil {
		return slog.Wrap(err)
	}
	_, err = conn.Do("SADD", notsByAlertKeyKey(ak), notification)
	return slog.Wrap(err)
}
Esempio n. 29
0
func (d *dataAccess) GetAllOpenIncidents() ([]*models.IncidentState, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetAllOpenIncidents"})()
	conn := d.GetConnection()
	defer conn.Close()

	// get open ids
	ids, err := int64s(conn.Do("HVALS", statesOpenIncidentsKey))
	if err != nil {
		return nil, slog.Wrap(err)
	}
	return d.incidentMultiGet(conn, ids)
}
Esempio n. 30
0
func (d *dataAccess) SetUnevaluated(ak models.AlertKey, uneval bool) error {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "SetUnevaluated"})()
	conn := d.GetConnection()
	defer conn.Close()

	op := "SREM"
	if uneval {
		op = "SADD"
	}
	_, err := conn.Do(op, statesUnevalKey(ak.Name()), ak)
	return slog.Wrap(err)
}