func testSarama(topic string, partition int32, seconds int) { stop := false config := sarama.NewClientConfig() client, err := sarama.NewClient("siesta", []string{"localhost:9092"}, config) if err != nil { panic(err) } messageChannel := make(chan *sarama.MessageSet, 10000) count := 0 go func() { for { set := <-messageChannel count += len(set.Messages) } }() broker, err := client.Leader(topic, partition) //warm up fmt.Println("warming up") for i := 0; i < 5; i++ { fetchRequest := new(sarama.FetchRequest) fetchRequest.MinBytes = 1 fetchRequest.MaxWaitTime = 100 fetchRequest.AddBlock(topic, partition, 0, 500) broker.Fetch("siesta", fetchRequest) } fmt.Println("warm up finished, starting") go func() { time.Sleep(time.Duration(seconds) * time.Second) stop = true }() offset := int64(0) if err != nil { panic(err) } for !stop { fetchRequest := new(sarama.FetchRequest) fetchRequest.MinBytes = 1 fetchRequest.MaxWaitTime = 100 fetchRequest.AddBlock(topic, partition, offset, 500) response, err := broker.Fetch("siesta", fetchRequest) if err != nil { panic(err) } set := response.Blocks[topic][partition].MsgSet messageChannel <- &set offset = set.Messages[len(set.Messages)-1].Offset } fmt.Printf("%d within %d secnods\n", count, seconds) fmt.Printf("%d average\n", count/seconds) }
// This will be called each time the fetch request to Kafka should be issued. Topic, partition and offset are self-explanatory. // Returns slice of Messages and an error if a fetch error occurred. func (this *SaramaClient) Fetch(topic string, partition int32, offset int64) ([]*Message, error) { leader, err := this.client.Leader(topic, partition) if err != nil { this.client.RefreshMetadata(topic) return nil, err } fetchRequest := new(sarama.FetchRequest) fetchRequest.MinBytes = this.config.FetchMinBytes fetchRequest.MaxWaitTime = this.config.FetchWaitMaxMs Debugf(this, "Adding block: topic=%s, partition=%d, offset=%d, fetchsize=%d", topic, partition, offset, this.config.FetchMessageMaxBytes) fetchRequest.AddBlock(topic, partition, offset, this.config.FetchMessageMaxBytes) response, err := leader.Fetch(fetchRequest) if err != nil { this.client.RefreshMetadata(topic) return nil, err } messages := make([]*Message, 0) if response != nil { Debug(this, "Processing fetch response") for topic, partitionAndData := range response.Blocks { for partition, data := range partitionAndData { switch data.Err { case sarama.ErrNoError: { if len(data.MsgSet.Messages) > 0 { this.filterPartitionData(data, offset) messages = this.collectMessages(data, topic, partition) if this.config.Debug { timestamp := time.Now().UnixNano() / int64(time.Millisecond) for _, message := range messages { message.DecodedKey = []int64{timestamp} } } } else { Debugf(this, "No messages in %s:%d at offset %d", topic, partition, offset) } } default: { this.client.RefreshMetadata(topic) return nil, data.Err } } } } } return messages, nil }