// 清空Error channel
func dumpErrors(p *sarama.Producer, timer chan bool) {
	for {
		select {
		case <-p.Errors():
		case <-timer:
			runtime.Gosched()
		}
	}
}
Beispiel #2
0
func (this *MirrorMaker) produceRoutine(producer *sarama.Producer, channelIndex int) {
	for msg := range this.messageChannels[channelIndex] {
		var key sarama.Encoder
		if !this.config.PreservePartitions {
			key = sarama.ByteEncoder(msg.Key)
		} else {
			key = Int32Encoder(msg.Partition)
		}
		producer.Input() <- &sarama.ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: key, Value: sarama.ByteEncoder(msg.Value)}
	}
}
// pushMetrics pushes the load balancer statistic to a Kafka Topic
func pushMetrics(producer *sarama.Producer, mode string) {

	// The list of metrics we want to filter out of the total stats dump from haproxy
	wantedMetrics := []string{"Scur", "Qcur", "Smax", "Slim", "Weight", "Qtime", "Ctime", "Rtime", "Ttime", "Req_rate", "Req_rate_max", "Req_tot", "Rate", "Rate_lim", "Rate_max"}

	// get metrics every second, for ever.
	for {

		stats, _ := GetStats("all")
		localTime := int64(time.Now().Unix())

		// for each proxy in the stats dump, pick out the wanted metrics, parse them and send 'm to Kafka
		for _, proxy := range stats {

			// filter out the metrics for haproxy's own stats page
			if proxy.Pxname != "stats" {

				// loop over all wanted metrics for the current proxy
				for _, metric := range wantedMetrics {

					fullMetricName := proxy.Pxname + "." + strings.ToLower(proxy.Svname) + "." + strings.ToLower(metric)
					field := reflect.ValueOf(proxy).FieldByName(metric).String()
					if field != "" {

						metricValue, _ := strconv.Atoi(field)

						metricObj := Metric{fullMetricName, metricValue, localTime}
						jsonObj, _ := json.MarshalIndent(metricObj, "", " ")

						err := producer.SendMessage(mode+"."+"all", sarama.StringEncoder("lbmetrics"), sarama.StringEncoder(jsonObj))
						if err != nil {

							log.Error("Error sending message to Kafka " + err.Error())

						} else {
							log.Debug("Successfully sent message to Kafka on topic: " + mode + "." + "all")
						}
					}
				}
			}
		}
		time.Sleep(3000 * time.Millisecond)
	}
}
Beispiel #4
0
func (this *SyslogProducer) produceRoutine(producer *sarama.Producer) {
	for msg := range this.incoming {
		Tracef(this, "Got message: %s", msg)
		producer.Input() <- this.config.Transformer(msg, this.config.Topic)
	}
}
// 异步发送msg
func publish(producer *sarama.Producer, topic string, msg string) error {
	return producer.QueueMessage(topic, nil, sarama.StringEncoder(msg))
	// log.Println(msg)
}