func testSarama(topic string, partition int32, seconds int) {
stop := false
config := sarama.NewClientConfig()
client, err := sarama.NewClient("siesta", []string{"localhost:9092"}, config)
if err != nil {
panic(err)
}
messageChannel := make(chan *sarama.MessageSet, 10000)
count := 0
go func() {
for {
set := <-messageChannel
count += len(set.Messages)
}
}()
broker, err := client.Leader(topic, partition)
//warm up
fmt.Println("warming up")
for i := 0; i < 5; i++ {
fetchRequest := new(sarama.FetchRequest)
fetchRequest.MinBytes = 1
fetchRequest.MaxWaitTime = 100
fetchRequest.AddBlock(topic, partition, 0, 500)
broker.Fetch("siesta", fetchRequest)
}
fmt.Println("warm up finished, starting")
go func() {
time.Sleep(time.Duration(seconds) * time.Second)
stop = true
}()
offset := int64(0)
if err != nil {
panic(err)
}
for !stop {
fetchRequest := new(sarama.FetchRequest)
fetchRequest.MinBytes = 1
fetchRequest.MaxWaitTime = 100
fetchRequest.AddBlock(topic, partition, offset, 500)
response, err := broker.Fetch("siesta", fetchRequest)
if err != nil {
panic(err)
}
set := response.Blocks[topic][partition].MsgSet
messageChannel <- &set
offset = set.Messages[len(set.Messages)-1].Offset
}
fmt.Printf("%d within %d secnods\n", count, seconds)
fmt.Printf("%d average\n", count/seconds)
}
// This will be called each time the fetch request to Kafka should be issued. Topic, partition and offset are self-explanatory.
// Returns slice of Messages and an error if a fetch error occurred.
func (this *SaramaClient) Fetch(topic string, partition int32, offset int64) ([]*Message, error) {
leader, err := this.client.Leader(topic, partition)
if err != nil {
this.client.RefreshMetadata(topic)
return nil, err
}
fetchRequest := new(sarama.FetchRequest)
fetchRequest.MinBytes = this.config.FetchMinBytes
fetchRequest.MaxWaitTime = this.config.FetchWaitMaxMs
Debugf(this, "Adding block: topic=%s, partition=%d, offset=%d, fetchsize=%d", topic, partition, offset, this.config.FetchMessageMaxBytes)
fetchRequest.AddBlock(topic, partition, offset, this.config.FetchMessageMaxBytes)
response, err := leader.Fetch(fetchRequest)
if err != nil {
this.client.RefreshMetadata(topic)
return nil, err
}
messages := make([]*Message, 0)
if response != nil {
Debug(this, "Processing fetch response")
for topic, partitionAndData := range response.Blocks {
for partition, data := range partitionAndData {
switch data.Err {
case sarama.ErrNoError:
{
if len(data.MsgSet.Messages) > 0 {
this.filterPartitionData(data, offset)
messages = this.collectMessages(data, topic, partition)
if this.config.Debug {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
for _, message := range messages {
message.DecodedKey = []int64{timestamp}
}
}
} else {
Debugf(this, "No messages in %s:%d at offset %d", topic, partition, offset)
}
}
default:
{
this.client.RefreshMetadata(topic)
return nil, data.Err
}
}
}
}
}
return messages, nil
}
func processPartition(command string, topic string, partition int32, controlChannel chan []int64, offsetManager sarama.OffsetManager) {
pom, err := offsetManager.ManagePartition(topic, int32(partition))
if err != nil {
exit(err)
}
consumerOffset, _ := pom.NextOffset()
offset, err := kafkaClient.GetOffset(topic, int32(partition), sarama.OffsetNewest)
//fmt.Println(topic, partition, consumerOffset, offset)
if err != nil {
exit(err)
}
var response = make([]int64, 0)
var timelag int64
lag := offset - consumerOffset + 1
response = append(response, lag)
if command == "lag_and_time" {
broker, err := kafkaClient.Leader(topic, partition)
if err != nil {
exit(err)
}
fetchRequest := sarama.FetchRequest{MaxWaitTime: 10000, MinBytes: 0}
fetchRequest.AddBlock(topic, partition, consumerOffset-2, 500000)
fetchResponse, err := broker.Fetch(&fetchRequest)
if err != nil {
exit(err)
}
block := fetchResponse.GetBlock(topic, partition)
messages := block.MsgSet.Messages
if len(messages) > 0 {
msg := messages[0].Messages()[0].Msg.Value
var decodedData map[string]interface{}
codec.NewDecoderBytes(msg, &msgpack).Decode(&decodedData)
timestamp := decodedData["timestamp"]
switch timestamp := timestamp.(type) {
case uint64:
timelag = int64(timestamp)
default:
fmt.Println(timestamp)
exit(errors.New("message is missing timestamp"))
}
} else {
timelag = 0
}
response = append(response, timelag)
}
controlChannel <- response
}