func NewKafka(numberOfMessages int, testLatency bool) *Kafka { config := sarama.NewConfig() client, _ := sarama.NewClient([]string{"localhost:9092"}, config) topic := "test" pub, _ := sarama.NewAsyncProducer([]string{"localhost:9092"}, config) consumer, _ := sarama.NewConsumerFromClient(client) sub, _ := consumer.ConsumePartition(topic, 0, sarama.OffsetNewest) var handler benchmark.MessageHandler if testLatency { handler = &benchmark.LatencyMessageHandler{ NumberOfMessages: numberOfMessages, Latencies: []float32{}, } } else { handler = &benchmark.ThroughputMessageHandler{NumberOfMessages: numberOfMessages} } return &Kafka{ handler: handler, client: client, pub: pub, sub: sub, topic: topic, } }
func CreateKafkaTopic() *KafkaTopic { client, err := sarama.NewClient([]string{"kafka:9092"}, sarama.NewConfig()) if err != nil { panic(err) } else { fmt.Printf("Kafka Client connected: %v\n", client) } topic := "http-request" producer, err := sarama.NewAsyncProducerFromClient(client) if err != nil { panic(err) } else { fmt.Printf("Kafka Producer connected: %v\n", producer) } producable := producer.Input() consumer, err := sarama.NewConsumerFromClient(client) if err != nil { panic(err) } else { fmt.Printf("Kafka Consumer connected: %v\n", consumer) } consumable, err := consumer.ConsumePartition(topic, 0, 0) if err != nil { panic(err) } return &KafkaTopic{client, topic, producer, producable, consumer, consumable} }
func getMasterConsumer(client sarama.Client) sarama.Consumer { //fmt.Println("Getting consumer.") master, err := sarama.NewConsumerFromClient(client) if err != nil { panic(err) } return master }
func runConsume(cmd *Command, args []string) { brokers := brokers() config := sarama.NewConfig() config.ClientID = "k consume" config.Consumer.Return.Errors = true client, err := sarama.NewClient(brokers, config) must(err) defer client.Close() consumer, err := sarama.NewConsumerFromClient(client) must(err) defer consumer.Close() signals := make(chan os.Signal, 1) defer close(signals) signal.Notify(signals, os.Interrupt) // calculate a starting offset _, newestOffset := offsets(client, topic, partition) must(err) startingOffset := newestOffset if offset < 0 { startingOffset = newestOffset + offset } else if offset > 0 { startingOffset = offset } // TODO: support consuming all partitions fmt.Fprintf(os.Stderr, "Using starting offset: %d\n", startingOffset) partConsumer, err := consumer.ConsumePartition(topic, partition, startingOffset) must(err) defer partConsumer.Close() var received, errors int consumerLoop: for { select { case msg := <-partConsumer.Messages(): if msg.Key != nil { fmt.Printf("%s\t", string(msg.Key)) } fmt.Println(string(msg.Value)) received++ if n > 0 && received >= n { break consumerLoop } case err := <-partConsumer.Errors(): fmt.Fprintf(os.Stderr, "Failed to receive message: %s\n", err) errors++ case <-signals: break consumerLoop } } fmt.Fprintf(os.Stderr, "Messages received: %d, errors: %d\n", received, errors) }
// Subscribe prepares the peer to consume messages. func (k *Peer) Subscribe() error { consumer, err := sarama.NewConsumerFromClient(k.client) if err != nil { return err } pc, err := consumer.ConsumePartition(topic, 0, sarama.OffsetNewest) if nil != err { return err } k.consumer = pc return nil }
func (this *Peek) simpleConsumeTopic(zkcluster *zk.ZkCluster, kfk sarama.Client, topic string, partitionId int32, msgCh chan *sarama.ConsumerMessage) { consumer, err := sarama.NewConsumerFromClient(kfk) if err != nil { panic(err) } defer consumer.Close() if partitionId == -1 { // all partitions partitions, err := kfk.Partitions(topic) if err != nil { panic(err) } for _, p := range partitions { offset := this.offset if this.lastN > 0 { latestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetNewest) swallow(err) oldestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetOldest) swallow(err) offset = latestOffset - this.lastN if offset < oldestOffset { offset = oldestOffset } if offset == 0 { // no message in store return } } go this.consumePartition(zkcluster, kfk, consumer, topic, p, msgCh, offset) } } else { offset := this.offset if this.lastN > 0 { latestOffset, err := kfk.GetOffset(topic, partitionId, sarama.OffsetNewest) swallow(err) offset = latestOffset - this.lastN if offset < 0 { offset = sarama.OffsetOldest } } this.consumePartition(zkcluster, kfk, consumer, topic, partitionId, msgCh, offset) } }
func New(kfkConn sarama.Client, transmitter Transmitter) (*ListenHandler, error) { consumer, err := sarama.NewConsumerFromClient(kfkConn) if err != nil { return nil, fmt.Errorf("Failed to create Kafka consumer: %v", err) } lh := &ListenHandler{ listenbus: make(chan chan<- listenRequest), transmitter: transmitter, consumer: consumer, kill: killchan.New(), dead: killchan.New(), } go lh.run() return lh, nil }
func main() { flag.Parse() if *verbose { sarama.Logger = logger } config := sarama.NewConfig() config.Producer.Return.Successes = true config.Producer.Return.Errors = true config.Consumer.Return.Errors = true client, err := sarama.NewClient(strings.Split(*brokerList, ","), config) if err != nil { logger.Fatalln("Failed to start Kafka client:", err) } defer func() { if err := client.Close(); err != nil { logger.Println("Failed to close client:", err) } }() producer, err := sarama.NewAsyncProducerFromClient(client) if err != nil { logger.Fatalln("Failed to start Kafka producer:", err) } consumer, err := sarama.NewConsumerFromClient(client) if err != nil { logger.Fatalln("Failed to start Kafka consumer:", err) } signal.Notify(shutdown, os.Interrupt, os.Kill, syscall.SIGHUP, syscall.SIGTERM) expectations := make(chan *sarama.ProducerMessage, ExpectationBufferSize) started := time.Now() var verifierWg sync.WaitGroup verifierWg.Add(2) go expectationProducer(producer, expectations, &verifierWg) go expectationConsumer(consumer, expectations, &verifierWg) verifierWg.Wait() stats.Print() logger.Println() logger.Printf("Done after %0.2fs.\n", float64(time.Since(started))/float64(time.Second)) }
// NewConsumer initializes a new consumer func NewConsumer(addrs []string, groupID string, topics []string, config *Config) (*Consumer, error) { if config == nil { config = NewConfig() } if err := config.Validate(); err != nil { return nil, err } client, err := sarama.NewClient(addrs, &config.Config) if err != nil { return nil, err } csmr, err := sarama.NewConsumerFromClient(client) if err != nil { _ = client.Close() return nil, err } c := &Consumer{ config: config, client: client, csmr: csmr, subs: newPartitionMap(), groupID: groupID, topics: topics, dying: make(chan none), dead: make(chan none), errors: make(chan error, config.ChannelBufferSize), messages: make(chan *sarama.ConsumerMessage, config.ChannelBufferSize), notifications: make(chan *Notification, 1), } if err := c.selectBroker(); err != nil { _ = client.Close() return nil, err } go c.mainLoop() return c, nil }
func (this *WatchAppError) consumeAppErrLogs(msgChan chan<- *sarama.ConsumerMessage) error { var ( cluster = os.Getenv("APPLOG_CLUSTER") topic = os.Getenv("APPLOG_TOPIC") ) if cluster == "" || topic == "" { return fmt.Errorf("empty cluster/topic params provided, kateway.apperr disabled") } zkcluster := this.Zkzone.NewCluster(cluster) brokerList := zkcluster.BrokerList() if len(brokerList) == 0 { return fmt.Errorf("cluster[%s] has empty brokers", cluster) } kfk, err := sarama.NewClient(brokerList, sarama.NewConfig()) if err != nil { return err } defer kfk.Close() consumer, err := sarama.NewConsumerFromClient(kfk) if err != nil { return err } defer consumer.Close() partitions, err := kfk.Partitions(topic) if err != nil { return err } var wg sync.WaitGroup for _, p := range partitions { wg.Add(1) go this.consumePartition(zkcluster, consumer, topic, p, sarama.OffsetNewest, msgChan, &wg) } wg.Wait() return nil }
func (f *File) reconsume(offset int64) error { if err := f.dir.reconnectKafkaIfNecessary(); err != nil { return err } consumer, err := sarama.NewConsumerFromClient(f.dir.Client) if err != nil { log.Error(err) return err } cp, err := consumer.ConsumePartition(f.topic, f.partitionId, offset) if err != nil { log.Error(err) return err } f.consumer = cp return nil }
func Consume() { config := sarama.NewConfig() //config.ChannelBufferSize = master, _ := sarama.NewClient([]string{"localhost:9095"}, config) cmas, _ := sarama.NewConsumerFromClient(master) offsetmanager, _ := sarama.NewOffsetManagerFromClient("test2", master) part_man, _ := offsetmanager.ManagePartition("tt.test", 1) //part_man.MarkOffset(0, "tt.test") off1, mata1 := part_man.NextOffset() fmt.Println(off1) fmt.Println(mata1) csm, _ := cmas.ConsumePartition("tt.test", 1, off1) //csm.Messages //Loop: for i := 0; i < 200; i++ { select { case message := <-csm.Messages(): off1++ part_man.MarkOffset(off1, "tt.test") fmt.Println(message) case err := <-csm.Errors(): fmt.Println(err) //csm.Close() // break Loop } } part_man.MarkOffset(off1, "tt.test") csm.Close() fmt.Println(off1) part_man.Close() //part_man2, _ := offsetmanager.ManagePartition("syslog-ng", 0) //off2, mata2 := part_man2.NextOffset() //part_man.MarkOffset(off2, "syslog-ng") //fmt.Println(off2) //fmt.Println(mata2) cmas.Close() master.Close() }
func NewKafkaClient(app *ApplicationContext, cluster string) (*KafkaClient, error) { // Set up sarama config from profile clientConfig := sarama.NewConfig() profile := app.Config.Clientprofile[app.Config.Kafka[cluster].Clientprofile] clientConfig.ClientID = profile.ClientID clientConfig.Net.TLS.Enable = profile.TLS clientConfig.Net.TLS.Config = &tls.Config{} clientConfig.Net.TLS.Config.InsecureSkipVerify = profile.TLSNoVerify sclient, err := sarama.NewClient(app.Config.Kafka[cluster].Brokers, clientConfig) if err != nil { return nil, err } // Create sarama master consumer master, err := sarama.NewConsumerFromClient(sclient) if err != nil { sclient.Close() return nil, err } client := &KafkaClient{ app: app, cluster: cluster, client: sclient, masterConsumer: master, requestChannel: make(chan *BrokerTopicRequest), messageChannel: make(chan *sarama.ConsumerMessage), errorChannel: make(chan *sarama.ConsumerError), wgFanIn: sync.WaitGroup{}, wgProcessor: sync.WaitGroup{}, topicMap: make(map[string]int), topicMapLock: sync.RWMutex{}, } // Start the main processor goroutines for __consumer_offset messages client.wgProcessor.Add(2) go func() { defer client.wgProcessor.Done() for msg := range client.messageChannel { go client.processConsumerOffsetsMessage(msg) } }() go func() { defer client.wgProcessor.Done() for err := range client.errorChannel { log.Errorf("Consume error on %s:%v: %v", err.Topic, err.Partition, err.Err) } }() // Start goroutine to handle topic metadata requests. Do this first because the getOffsets call needs this working client.RefreshTopicMap() go func() { for r := range client.requestChannel { client.getPartitionCount(r) } }() // Now get the first set of offsets and start a goroutine to continually check them client.getOffsets() client.brokerOffsetTicker = time.NewTicker(time.Duration(client.app.Config.Tickers.BrokerOffsets) * time.Second) go func() { for _ = range client.brokerOffsetTicker.C { client.getOffsets() } }() // Get a partition count for the consumption topic partitions, err := client.client.Partitions(client.app.Config.Kafka[client.cluster].OffsetsTopic) if err != nil { return nil, err } // Start consumers for each partition with fan in client.partitionConsumers = make([]sarama.PartitionConsumer, len(partitions)) log.Infof("Starting consumers for %v partitions of %s in cluster %s", len(partitions), client.app.Config.Kafka[client.cluster].OffsetsTopic, client.cluster) for i, partition := range partitions { pconsumer, err := client.masterConsumer.ConsumePartition(client.app.Config.Kafka[client.cluster].OffsetsTopic, partition, sarama.OffsetNewest) if err != nil { return nil, err } client.partitionConsumers[i] = pconsumer client.wgFanIn.Add(2) go func() { defer client.wgFanIn.Done() for msg := range pconsumer.Messages() { client.messageChannel <- msg } }() go func() { defer client.wgFanIn.Done() for err := range pconsumer.Errors() { client.errorChannel <- err } }() } return client, nil }
func main() { flag.Parse() if *brokers == "" { flag.PrintDefaults() os.Exit(1) } brokerList := strings.Split(*brokers, ",") cfg := sarama.NewConfig() kafkaCllient, e := sarama.NewClient(brokerList, cfg) if e != nil { panic(e) } consumer, e := sarama.NewConsumerFromClient(kafkaCllient) if e != nil { panic(e) } pConsumer, _ := consumer.ConsumePartition("sensors", 0, 0) u, _ := url.Parse("http://localhost:8086") influxConfig := influx.Config{ URL: *u, Username: "******", Password: "******", } influxClient, _ := influx.NewClient(influxConfig) influxClient.Ping() for msg := range pConsumer.Messages() { // Decode the SensorData from protobuf string data. data := &sensory_proto.SensorData{} proto.Unmarshal(msg.Value, data) pts := make([]influx.Point, len(data.Reads)) for i, read := range data.Reads { readType := typeToString[read.Type] var tm time.Time if read.Timestamp == 0 { tm = time.Now() } else { print(intVal) tm = time.Unix(int64(read.Timestamp), 0) } fields := make(map[string]interface{}) var ( intVal *int32 = &read.Intval fVal *float32 = &read.Floatval ) if intVal != nil { fields["value"] = *intVal } if fVal != nil { fields["value"] = *fVal } if sender, ok := senderToString[data.SenderId]; ok { fields["sender"] = sender } point := influx.Point{ Measurement: readType, Fields: fields, Tags: map[string]string{}, Time: tm, } pts[i] = point fmt.Printf("Sender ID %d type(%s) int(%d) float(%f)\n", data.SenderId, readType, read.Intval, read.Floatval) } bps := influx.BatchPoints{ Points: pts, Database: "sensors", RetentionPolicy: "default", } _, err := influxClient.Write(bps) if err != nil { print("ERROR") fmt.Println(err) } } }
func main() { flag.Parse() if *verbose { sarama.Logger = log.New(os.Stdout, "[Sarama]", log.LstdFlags) } brokers := getBrokers(*brokerIP, *brokerPorts) config := sarama.NewConfig() config.Consumer.Return.Errors = true client, err := sarama.NewClient(brokers, config) if err != nil { logger.Fatal(err) } defer func() { if err := client.Close(); err != nil { logger.Fatal(err) } }() topic := "my_topic" partitions, err := client.Partitions(topic) if err != nil { logger.Fatal(err) } master, err := sarama.NewConsumerFromClient(client) if err != nil { logger.Fatal(err) } defer func() { if err := master.Close(); err != nil { logger.Fatal(err) } }() var wg sync.WaitGroup wg.Add(len(partitions)) for _, partition := range partitions { go func(master sarama.Consumer, partition int32) { defer wg.Done() consumer, err := master.ConsumePartition(topic, partition, sarama.OffsetNewest) if err != nil { logger.Fatal(err) } signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt) // Count how many message processed msgCount := 0 // Get signnal for finish doneCh := make(chan struct{}) go func() { for { select { case err := <-consumer.Errors(): logger.Println(err) case msg := <-consumer.Messages(): msgCount++ logger.Println("Received messages", string(msg.Key), string(msg.Value)) case <-signals: logger.Println("Interrupt is detected") doneCh <- struct{}{} } } }() <-doneCh fmt.Println("Processed", msgCount, "messages") }(master, partition) } wg.Wait() }