func partitionExpectationConsumer(pc sarama.PartitionConsumer, expectations <-chan *sarama.ProducerMessage, wg *sync.WaitGroup) {
defer wg.Done()
defer func() {
if err := pc.Close(); err != nil {
logger.Println("Failed to close partitionconsumer:", err)
}
}()
for expectation := range expectations {
msg := <-pc.Messages()
if msg.Offset != expectation.Offset {
fmt.Printf("Unexpected offset %d!\n", msg.Offset)
}
key, _ := expectation.Key.Encode()
if string(key) != string(msg.Key) {
fmt.Printf("Unexpected key: %v!\n", msg.Key)
}
if msg.Value != nil {
fmt.Printf("Unexpected value: %v!\n", msg.Value)
}
stats.LogConsumed(expectation)
}
}
func (suite *KafkaTester) Test01() {
t := suite.T()
assert := assert.New(t)
const M1 = "message one"
const M2 = "message two"
var producer sarama.AsyncProducer
var consumer sarama.Consumer
var partitionConsumer sarama.PartitionConsumer
var err error
topic := makeTopicName()
{
config := sarama.NewConfig()
config.Producer.Return.Successes = false
config.Producer.Return.Errors = false
producer, err = sarama.NewAsyncProducer([]string{suite.server}, config)
assert.NoError(err)
defer close(t, producer)
producer.Input() <- &sarama.ProducerMessage{
Topic: topic,
Key: nil,
Value: sarama.StringEncoder(M1)}
producer.Input() <- &sarama.ProducerMessage{
Topic: topic,
Key: nil,
Value: sarama.StringEncoder(M2)}
}
{
consumer, err = sarama.NewConsumer([]string{suite.server}, nil)
assert.NoError(err)
defer close(t, consumer)
partitionConsumer, err = consumer.ConsumePartition(topic, 0, 0)
assert.NoError(err)
defer close(t, partitionConsumer)
}
{
mssg1 := <-partitionConsumer.Messages()
//t.Logf("Consumed: offset:%d value:%v", mssg1.Offset, string(mssg1.Value))
mssg2 := <-partitionConsumer.Messages()
//t.Logf("Consumed: offset:%d value:%v", mssg2.Offset, string(mssg2.Value))
assert.EqualValues(M1, string(mssg1.Value))
assert.EqualValues(M2, string(mssg2.Value))
}
}
func handleConsumer(consumer kafka.PartitionConsumer) {
for message := range consumer.Messages() {
log.V(2).Printf("Got message: %v\n", message)
nextRoundRobinClient().inbox <- messages.Message{
Value: message.Value,
Partition: message.Partition,
Offset: message.Offset,
IsAckRequest: false,
}
}
}
// Consume kafka messages
func consumeMessages(producer kafka.AsyncProducer, consumer kafka.PartitionConsumer) error {
select {
case message := <-consumer.Messages():
var dat map[string]interface{}
if err := json.Unmarshal(message.Value, &dat); err != nil {
fmt.Printf("Error with Unmarshal: %s\n", err)
} else {
monascaTS, monascaUS, monascaSS := createMonascaSpec(dat)
mtm, mum, msm := marshalMonasca(monascaTS, monascaUS, monascaSS)
produceMonascaMetrics(producer, dat, mtm, mum, msm)
}
case err := <-consumer.Errors():
return err
}
return nil
}
func consumePartitionLoop(
closer chan struct{},
out chan string,
pc sarama.PartitionConsumer,
p int32,
end int64,
) {
for {
timeout := make(<-chan time.Time)
if config.consume.timeout > 0 {
timeout = time.After(config.consume.timeout)
}
select {
case <-timeout:
log.Printf("Consuming from partition [%v] timed out.", p)
pc.Close()
return
case <-closer:
pc.Close()
return
case msg, ok := <-pc.Messages():
if ok {
m := consumedMessage{
Partition: msg.Partition,
Offset: msg.Offset,
}
k := string(msg.Key)
if msg.Key != nil {
m.Key = &k
}
v := string(msg.Value)
if msg.Value != nil {
m.Value = &v
}
byts, err := json.Marshal(m)
if err != nil {
fmt.Fprintf(os.Stderr, "Quitting due to unexpected error during marshal: %v\n", err)
close(closer)
return
}
out <- string(byts)
}
if end > 0 && msg.Offset >= end {
pc.Close()
return
}
}
}
}