Esempio n. 1
0
func (p *Pipeline) ProcessIncident(in *event.Incident) {

	// start tracking this incident in memory so we can call back to it
	p.tracker.TrackIncident(in)

	// dedup the incident
	if p.Dedupe(in) {

		// update the incident in the index
		if in.Status != event.OK {
			p.index.PutIncident(in)
		} else {
			p.index.DeleteIncidentById(in.IndexName())
		}

		// fetch the escalation to take
		esc, ok := p.escalations.Collection()[in.Escalation]
		if ok {

			// send to every alarm in the escalation
			for _, a := range esc {
				a.Send(in)
			}
		} else {
			logrus.Error("unknown escalation", in.Escalation)
		}
	}
}
Esempio n. 2
0
func (t *Tracker) GetIncidentResolver(i *event.Incident) chan *event.Incident {
	var res chan *event.Incident
	t.Query(func(r *Tracker) {
		res, _ = r.incidentResolvers[string(i.IndexName())]
	})
	return res
}
Esempio n. 3
0
// Send an email via smtp
func (e *Email) Send(i *event.Incident) error {

	//For now set the description as both the subject and body
	headers := make(map[string]string)
	headers["From"] = e.conf.Sender
	headers["To"] = strings.Join(e.conf.Recipients, ",")
	headers["Subject"] = i.FormatDescription()
	headers["MIME-Version"] = "1.0"
	headers["Content-Type"] = "text/plain; charset=\"utf-8\""
	headers["Content-Transfer-Encoding"] = "base64"

	// make the body a json encoded representation of the incidnent
	body, err := json.MarshalIndent(i, "", "    ")
	if err != nil {
		logrus.Errorf("Unable to encode incidnet for email %s", err.Error())
	}

	log.Println("sending email")
	err = smtp.SendMail(e.conf.Host+":"+strconv.Itoa(e.conf.Port), *e.Auth,
		e.conf.Sender, e.conf.Recipients, []byte(writeEmailBuffer(headers, string(body))))
	if err != nil {
		logrus.Errorf("Unable to send email via smtp %s", err)
	}
	log.Println("done sending mail")
	return err
}
Esempio n. 4
0
// returns true if this is a new incident, false if it is a duplicate
func (p *Pipeline) Dedupe(i *event.Incident) bool {
	old := p.index.GetIncident(i.IndexName())

	if old == nil {
		return i.Status != event.OK
	}

	return old.Status != i.Status
}
Esempio n. 5
0
// PassIncident takes an incident into the escalation for processing
func (e *EscalationPolicy) PassIncident(i *event.Incident) {

	// only process incidents that this policy subscribes to
	if e.isSubscribed(i) {

		// send if off to every escalation known about
		for _, ep := range e.Escalations {
			err := ep.Send(i)
			if err != nil {
				logrus.Errorf("Unable to forward incident %s to escalation %+v", i.FormatDescription(), ep)
			}
		}
	}
}
Esempio n. 6
0
func (p *Pipeline) PutIncident(in *event.Incident) {
	if in.Id == 0 {
		in.Id = p.index.GetIncidentCounter()
		p.index.UpdateIncidentCounter(in.Id + 1)
	}
	p.index.PutIncident(in)
}
Esempio n. 7
0
// TrackIncident will allow the tracker to keep state about an incident
func (t *Tracker) TrackIncident(i *event.Incident) {
	if i.GetResolve() != nil {
		t.Query(func(r *Tracker) {

			// Don't keep track of "OK" incident resolvers, as ok's can't be resolved
			if i.Status != event.OK {
				r.incidentResolvers[string(i.IndexName())] = i.GetResolve()
			}
			r.totalIncidents.inc()
		})
	}
}
Esempio n. 8
0
func (c *Console) Send(i *event.Incident) error {

	switch i.Status {
	case event.OK:
		logrus.Info(i.FormatDescription())
	case event.WARNING:
		logrus.Warn(i.FormatDescription())
	case event.CRITICAL:
		logrus.Error(i.FormatDescription())
	}

	return nil
}
Esempio n. 9
0
func (p *PagerDuty) Send(i *event.Incident) error {
	var pdPevent *pagerduty.Event
	switch i.Status {
	case event.CRITICAL, event.WARNING:
		pdPevent = pagerduty.NewTriggerEvent(p.conf.Key, i.FormatDescription())
	case event.OK:
		pdPevent = pagerduty.NewResolveEvent(p.conf.Key, i.FormatDescription())
	}
	pdPevent.IncidentKey = string(string(i.IndexName()))

	_, _, err := pagerduty.Submit(pdPevent)
	return err
}
Esempio n. 10
0
// processIncident forwards a deduped incident on to every escalation
func (p *Pipeline) processIncident(in *event.Incident) {
	in.GetEvent().SetState(event.StateIncident)

	// start tracking this incident in memory so we can call back to it
	p.tracker.TrackIncident(in)

	// dedup the incident
	if p.Dedupe(in) {

		// update the incident in the index
		if in.Status != event.OK {
			p.index.PutIncident(in)
		} else {
			p.index.DeleteIncidentById(in.IndexName())
		}

		// send it on to every escalation
		for _, esc := range p.escalations {
			esc.PassIncident(in)
		}
	}

	in.GetEvent().SetState(event.StateComplete)
}
// bangarang.annotation.{status}.{host}.{service}
func formatName(i *event.Incident) string {
	return strings.Replace(fmt.Sprintf("%s.%s.%s.%s", ANNOTATION_PREFIX, event.Status(i.Status), i.Tags.Get("host"), strings.Replace(i.FormatDescription(), ".", "_", -1)), " ", "_", -1)
}
Esempio n. 12
0
func (t *testingPasser) PassIncident(i *event.Incident) {
	if t.incidents == nil {
		t.incidents = map[string]*event.Incident{}
	}
	t.incidents[string(i.IndexName())] = i
}