func (this *Peek) consumePartition(zkcluster *zk.ZkCluster, kfk sarama.Client, consumer sarama.Consumer,
topic string, partitionId int32, msgCh chan *sarama.ConsumerMessage, offset int64) {
p, err := consumer.ConsumePartition(topic, partitionId, offset)
if err != nil {
this.Ui.Error(fmt.Sprintf("%s %s/%d: offset=%d %v", zkcluster.Name(), topic, partitionId, offset, err))
return
}
defer p.Close()
n := int64(0)
for {
select {
case <-this.quit:
return
case msg := <-p.Messages():
msgCh <- msg
n++
if this.lastN > 0 && n >= this.lastN {
return
}
}
}
}
func (this *WatchAppError) consumePartition(zkcluster *zk.ZkCluster, consumer sarama.Consumer,
topic string, partitionId int32, offset int64, msgCh chan<- *sarama.ConsumerMessage, wg *sync.WaitGroup) {
defer wg.Done()
p, err := consumer.ConsumePartition(topic, partitionId, offset)
if err != nil {
log.Error("%s %s/%d: offset=%d %v", zkcluster.Name(), topic, partitionId, offset, err)
return
}
defer p.Close()
for {
select {
case <-this.Stop:
return
case err := <-p.Errors():
log.Critical("cluster[%s] %s/%d: %s", zkcluster.Name(), topic, partitionId, err)
return
case msg := <-p.Messages():
if msg != nil && this.predicate(msg.Value) {
msgCh <- msg
}
}
}
}
func (suite *KafkaTester) Test01() {
t := suite.T()
assert := assert.New(t)
const M1 = "message one"
const M2 = "message two"
var producer sarama.AsyncProducer
var consumer sarama.Consumer
var partitionConsumer sarama.PartitionConsumer
var err error
topic := makeTopicName()
{
config := sarama.NewConfig()
config.Producer.Return.Successes = false
config.Producer.Return.Errors = false
producer, err = sarama.NewAsyncProducer([]string{suite.server}, config)
assert.NoError(err)
defer close(t, producer)
producer.Input() <- &sarama.ProducerMessage{
Topic: topic,
Key: nil,
Value: sarama.StringEncoder(M1)}
producer.Input() <- &sarama.ProducerMessage{
Topic: topic,
Key: nil,
Value: sarama.StringEncoder(M2)}
}
{
consumer, err = sarama.NewConsumer([]string{suite.server}, nil)
assert.NoError(err)
defer close(t, consumer)
partitionConsumer, err = consumer.ConsumePartition(topic, 0, 0)
assert.NoError(err)
defer close(t, partitionConsumer)
}
{
mssg1 := <-partitionConsumer.Messages()
//t.Logf("Consumed: offset:%d value:%v", mssg1.Offset, string(mssg1.Value))
mssg2 := <-partitionConsumer.Messages()
//t.Logf("Consumed: offset:%d value:%v", mssg2.Offset, string(mssg2.Value))
assert.EqualValues(M1, string(mssg1.Value))
assert.EqualValues(M2, string(mssg2.Value))
}
}
func newPartitionConsumer(manager sarama.Consumer, topic string, partition int32, info offsetInfo, defaultOffset int64) (*partitionConsumer, error) {
pcm, err := manager.ConsumePartition(topic, partition, info.NextOffset(defaultOffset))
// Resume from default offset, if requested offset is out-of-range
if err == sarama.ErrOffsetOutOfRange {
info.Offset = -1
pcm, err = manager.ConsumePartition(topic, partition, defaultOffset)
}
if err != nil {
return nil, err
}
return &partitionConsumer{
pcm: pcm,
state: partitionState{Info: info},
dying: make(chan none),
dead: make(chan none),
}, nil
}
func expectationConsumer(c sarama.Consumer, expectations <-chan *sarama.ProducerMessage, wg *sync.WaitGroup) {
defer wg.Done()
defer func() {
if err := c.Close(); err != nil {
logger.Println("Failed to close consumer:", err)
}
}()
var (
partitionVerifiers = make(map[int32]*partitionVerifier)
consumerWg sync.WaitGroup
)
for expectation := range expectations {
partition := expectation.Partition
if partitionVerifiers[partition] == nil {
logger.Printf("Starting message verifier for partition %d...\n", partition)
pc, err := c.ConsumePartition(*topic, partition, expectation.Offset)
if err != nil {
logger.Fatalf("Failed to open partition consumer for %s/%d: %s", *topic, expectation.Partition, err)
}
partitionExpectations := make(chan *sarama.ProducerMessage)
partitionVerifiers[partition] = &partitionVerifier{pc: pc, expectations: partitionExpectations}
consumerWg.Add(1)
go partitionExpectationConsumer(pc, partitionExpectations, &consumerWg)
}
partitionVerifiers[partition].expectations <- expectation
}
for _, pv := range partitionVerifiers {
close(pv.expectations)
}
consumerWg.Wait()
}
func consumePartition(
config consumeConfig,
closer chan struct{},
out chan string,
consumer sarama.Consumer,
partition int32,
) {
offsets, ok := config.offsets[partition]
if !ok {
offsets, ok = config.offsets[-1]
}
start, err := offsets.start.value(clientMaker, partition)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to read start offset for partition %v err=%v\n", partition, err)
return
}
end, err := offsets.end.value(clientMaker, partition)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to read end offset for partition %v err=%v\n", partition, err)
return
}
partitionConsumer, err := consumer.ConsumePartition(
config.topic,
partition,
start,
)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to consume partition %v err=%v\n", partition, err)
return
}
consumePartitionLoop(closer, out, partitionConsumer, partition, end)
}