func testSarama(topic string, partition int32, seconds int) {
stop := false
config := sarama.NewClientConfig()
client, err := sarama.NewClient("siesta", []string{"localhost:9092"}, config)
if err != nil {
panic(err)
}
messageChannel := make(chan *sarama.MessageSet, 10000)
count := 0
go func() {
for {
set := <-messageChannel
count += len(set.Messages)
}
}()
broker, err := client.Leader(topic, partition)
//warm up
fmt.Println("warming up")
for i := 0; i < 5; i++ {
fetchRequest := new(sarama.FetchRequest)
fetchRequest.MinBytes = 1
fetchRequest.MaxWaitTime = 100
fetchRequest.AddBlock(topic, partition, 0, 500)
broker.Fetch("siesta", fetchRequest)
}
fmt.Println("warm up finished, starting")
go func() {
time.Sleep(time.Duration(seconds) * time.Second)
stop = true
}()
offset := int64(0)
if err != nil {
panic(err)
}
for !stop {
fetchRequest := new(sarama.FetchRequest)
fetchRequest.MinBytes = 1
fetchRequest.MaxWaitTime = 100
fetchRequest.AddBlock(topic, partition, offset, 500)
response, err := broker.Fetch("siesta", fetchRequest)
if err != nil {
panic(err)
}
set := response.Blocks[topic][partition].MsgSet
messageChannel <- &set
offset = set.Messages[len(set.Messages)-1].Offset
}
fmt.Printf("%d within %d secnods\n", count, seconds)
fmt.Printf("%d average\n", count/seconds)
}
// This will be called each time the fetch request to Kafka should be issued. Topic, partition and offset are self-explanatory.
// Returns slice of Messages and an error if a fetch error occurred.
func (this *SaramaClient) Fetch(topic string, partition int32, offset int64) ([]*Message, error) {
leader, err := this.client.Leader(topic, partition)
if err != nil {
this.client.RefreshMetadata(topic)
return nil, err
}
fetchRequest := new(sarama.FetchRequest)
fetchRequest.MinBytes = this.config.FetchMinBytes
fetchRequest.MaxWaitTime = this.config.FetchWaitMaxMs
Debugf(this, "Adding block: topic=%s, partition=%d, offset=%d, fetchsize=%d", topic, partition, offset, this.config.FetchMessageMaxBytes)
fetchRequest.AddBlock(topic, partition, offset, this.config.FetchMessageMaxBytes)
response, err := leader.Fetch(fetchRequest)
if err != nil {
this.client.RefreshMetadata(topic)
return nil, err
}
messages := make([]*Message, 0)
if response != nil {
Debug(this, "Processing fetch response")
for topic, partitionAndData := range response.Blocks {
for partition, data := range partitionAndData {
switch data.Err {
case sarama.ErrNoError:
{
if len(data.MsgSet.Messages) > 0 {
this.filterPartitionData(data, offset)
messages = this.collectMessages(data, topic, partition)
if this.config.Debug {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
for _, message := range messages {
message.DecodedKey = []int64{timestamp}
}
}
} else {
Debugf(this, "No messages in %s:%d at offset %d", topic, partition, offset)
}
}
default:
{
this.client.RefreshMetadata(topic)
return nil, data.Err
}
}
}
}
}
return messages, nil
}