// AmqpExecutor reads jobs from rabbitmq, executes them, and acknowledges them
// if they processed succesfully or encountered a fatal error
// (i.e. an error that we know won't recover on future retries, so no point in retrying)
func AmqpExecutor(fn GraphiteReturner, consumer rabbitmq.Consumer, cache *lru.Cache) {
executorNum.Inc(1)
defer executorNum.Dec(1)
consumer.Consume(func(msg *amqp.Delivery) error {
job := Job{}
if err := json.Unmarshal(msg.Body, &job); err != nil {
log.Error(0, "failed to unmarshal msg body.", err)
return nil
}
job.StoreMetricFunc = api.StoreMetric
var err error
if setting.AlertingInspect {
inspect(GraphiteAuthContextReturner, &job, cache)
} else {
err = execute(GraphiteAuthContextReturner, &job, cache)
}
if err != nil {
if strings.HasPrefix(err.Error(), "fatal:") {
log.Error(0, "%s: removing job from queue", err.Error())
return nil
}
log.Error(0, "%s: not acking message. retry later", err.Error())
}
return err
})
}
func InitCollectorController(metrics met.Backend) {
sec := setting.Cfg.Section("event_publisher")
cmd := &m.ClearCollectorSessionCommand{
InstanceId: setting.InstanceId,
}
if err := bus.Dispatch(cmd); err != nil {
log.Fatal(0, "failed to clear collectorSessions", err)
}
if sec.Key("enabled").MustBool(false) {
url := sec.Key("rabbitmq_url").String()
exchange := sec.Key("exchange").String()
exch := rabbitmq.Exchange{
Name: exchange,
ExchangeType: "topic",
Durable: true,
}
q := rabbitmq.Queue{
Name: "",
Durable: false,
AutoDelete: true,
Exclusive: true,
}
consumer := rabbitmq.Consumer{
Url: url,
Exchange: &exch,
Queue: &q,
BindingKey: []string{"INFO.monitor.*", "INFO.collector.*"},
}
err := consumer.Connect()
if err != nil {
log.Fatal(0, "failed to start event.consumer.", err)
}
consumer.Consume(eventConsumer)
} else {
//tap into the update/add/Delete events emitted when monitors are modified.
bus.AddEventListener(EmitUpdateMonitor)
bus.AddEventListener(EmitAddMonitor)
bus.AddEventListener(EmitDeleteMonitor)
bus.AddEventListener(HandleCollectorConnected)
bus.AddEventListener(HandleCollectorDisconnected)
}
metricsRecvd = metrics.NewCount("collector-ctrl.metrics-recv")
bufCh = make(chan m.MetricDefinition, runtime.NumCPU()*100)
go metricpublisher.ProcessBuffer(bufCh)
}
func distributed(url string, cache *lru.Cache) error {
exchange := "alertingJobs"
exch := rabbitmq.Exchange{
Name: exchange,
ExchangeType: "x-consistent-hash",
Durable: true,
}
if setting.EnableScheduler {
publisher := &rabbitmq.Publisher{Url: url, Exchange: &exch}
err := publisher.Connect()
if err != nil {
return err
}
jobQueue := newPreAMQPJobQueue(setting.PreAMQPJobQueueSize, publisher)
go Dispatcher(jobQueue)
}
q := rabbitmq.Queue{
Name: "",
Durable: false,
AutoDelete: true,
Exclusive: true,
}
for i := 0; i < setting.Executors; i++ {
consumer := rabbitmq.Consumer{
Url: url,
Exchange: &exch,
Queue: &q,
BindingKey: []string{"10"}, //consistant hashing weight.
}
if err := consumer.Connect(); err != nil {
log.Fatal(0, "failed to start event.consumer.", err)
}
AmqpExecutor(GraphiteAuthContextReturner, consumer, cache)
}
return nil
}
func InitCollectorController() {
sec := setting.Cfg.Section("event_publisher")
// get our instance-id
dataPath := setting.DataPath + "/instance-id"
log.Info("instance-id path: " + dataPath)
fs, err := os.Open(dataPath)
if err != nil {
fs, err = os.Create(dataPath)
if err != nil {
log.Fatal(0, "failed to create instance-id file", err)
}
defer fs.Close()
instanceId = uuid.NewV4().String()
if _, err := fs.Write([]byte(instanceId)); err != nil {
log.Fatal(0, "failed to write instanceId to file", err)
}
} else {
defer fs.Close()
content, err := ioutil.ReadAll(fs)
if err != nil {
log.Fatal(0, "failed to read instanceId", err)
}
instanceId = strings.Split(string(content), "\n")[0]
}
if instanceId == "" {
log.Fatal(0, "invalid instanceId. check "+dataPath, nil)
}
cmd := &m.ClearCollectorSessionCommand{
InstanceId: instanceId,
}
if err := bus.Dispatch(cmd); err != nil {
log.Fatal(0, "failed to clear collectorSessions", err)
}
if sec.Key("enabled").MustBool(false) {
url := sec.Key("rabbitmq_url").String()
exchange := sec.Key("exchange").String()
exch := rabbitmq.Exchange{
Name: exchange,
ExchangeType: "topic",
Durable: true,
}
q := rabbitmq.Queue{
Name: "",
Durable: false,
AutoDelete: true,
Exclusive: true,
}
consumer := rabbitmq.Consumer{
Url: url,
Exchange: &exch,
Queue: &q,
BindingKey: []string{"INFO.monitor.*", "INFO.collector.*"},
}
err := consumer.Connect()
if err != nil {
log.Fatal(0, "failed to start event.consumer.", err)
}
consumer.Consume(eventConsumer)
} else {
//tap into the update/add/Delete events emitted when monitors are modified.
bus.AddEventListener(EmitUpdateMonitor)
bus.AddEventListener(EmitAddMonitor)
bus.AddEventListener(EmitDeleteMonitor)
bus.AddEventListener(HandleCollectorConnected)
bus.AddEventListener(HandleCollectorDisconnected)
}
bufCh = make(chan m.MetricDefinition, runtime.NumCPU())
go metricpublisher.ProcessBuffer(bufCh)
}
func distributed(url string) error {
exchange := "alertingJobs"
exch := rabbitmq.Exchange{
Name: exchange,
ExchangeType: "x-consistent-hash",
Durable: true,
}
publisher := &rabbitmq.Publisher{Url: url, Exchange: &exch}
err := publisher.Connect()
if err != nil {
return err
}
jobQueue := make(chan Job, jobQueueSize)
go Dispatcher(jobQueue)
//send dispatched jobs to rabbitmq.
go func(jobQueue <-chan Job) {
for job := range jobQueue {
routingKey := fmt.Sprintf("%d", job.MonitorId)
msg, err := json.Marshal(job)
//log.Info("sending: " + string(msg))
if err != nil {
log.Error(3, "failed to marshal job to json.", err)
continue
}
publisher.Publish(routingKey, msg)
}
}(jobQueue)
q := rabbitmq.Queue{
Name: "",
Durable: false,
AutoDelete: true,
Exclusive: true,
}
consumer := rabbitmq.Consumer{
Url: url,
Exchange: &exch,
Queue: &q,
BindingKey: []string{"10"}, //consistant hashing weight.
}
if err := consumer.Connect(); err != nil {
log.Fatal(0, "failed to start event.consumer.", err)
}
consumeQueue := make(chan Job, jobQueueSize)
//read jobs from rabbitmq and push them into the execution channel.
consumer.Consume(func(msg *amqp.Delivery) error {
//convert from json to Job
job := Job{}
//log.Info("recvd: " + string(msg.Body))
if err := json.Unmarshal(msg.Body, &job); err != nil {
log.Error(0, "failed to unmarshal msg body.", err)
return err
}
job.StoreMetricFunc = api.StoreMetric
select {
case consumeQueue <- job:
default:
// TODO: alert when this happens
Stat.Increment("alert-dispatcher.jobs-skipped-due-to-slow-jobqueue")
}
return nil
})
//start group of workers to execute the jobs in the execution channel.
for i := 0; i < 10; i++ {
go Executor(GraphiteAuthContextReturner, consumeQueue)
}
return nil
}