func (this *Peek) simpleConsumeTopic(zkcluster *zk.ZkCluster, kfk sarama.Client, topic string, partitionId int32,
msgCh chan *sarama.ConsumerMessage) {
consumer, err := sarama.NewConsumerFromClient(kfk)
if err != nil {
panic(err)
}
defer consumer.Close()
if partitionId == -1 {
// all partitions
partitions, err := kfk.Partitions(topic)
if err != nil {
panic(err)
}
for _, p := range partitions {
offset := this.offset
if this.lastN > 0 {
latestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetNewest)
swallow(err)
oldestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetOldest)
swallow(err)
offset = latestOffset - this.lastN
if offset < oldestOffset {
offset = oldestOffset
}
if offset == 0 {
// no message in store
return
}
}
go this.consumePartition(zkcluster, kfk, consumer, topic, p, msgCh, offset)
}
} else {
offset := this.offset
if this.lastN > 0 {
latestOffset, err := kfk.GetOffset(topic, partitionId, sarama.OffsetNewest)
swallow(err)
offset = latestOffset - this.lastN
if offset < 0 {
offset = sarama.OffsetOldest
}
}
this.consumePartition(zkcluster, kfk, consumer, topic, partitionId, msgCh, offset)
}
}
func readTopic(client sarama.Client, name string) (topic, error) {
t := topic{Name: name}
if config.topic.partitions {
ps, err := client.Partitions(name)
if err != nil {
return t, err
}
for _, p := range ps {
np := partition{Id: p}
oldest, err := client.GetOffset(name, p, sarama.OffsetOldest)
if err != nil {
return t, err
}
np.OldestOffset = oldest
newest, err := client.GetOffset(name, p, sarama.OffsetNewest)
if err != nil {
return t, err
}
np.NewestOffset = newest
if config.topic.leaders {
b, err := client.Leader(name, p)
if err != nil {
return t, err
}
np.Leader = b.Addr()
}
if config.topic.replicas {
rs, err := client.Replicas(name, p)
if err != nil {
return t, err
}
np.Replicas = rs
}
t.Partitions = append(t.Partitions, np)
}
}
return t, nil
}
Expect(os.MkdirAll(testKafkaData, 0777)).NotTo(HaveOccurred())
Expect(testZkCmd.Start()).NotTo(HaveOccurred())
Expect(testKafkaCmd.Start()).NotTo(HaveOccurred())
// Wait for client
Eventually(func() error {
var err error
testClient, err = sarama.NewClient(testKafkaAddrs, nil)
return err
}, "10s", "1s").ShouldNot(HaveOccurred())
// Ensure we can retrieve partition info
Eventually(func() error {
_, err := testClient.Partitions(testTopics[0])
return err
}, "10s", "500ms").ShouldNot(HaveOccurred())
// Seed a few messages
Expect(testSeed(1000)).NotTo(HaveOccurred())
})
var _ = AfterSuite(func() {
_ = testClient.Close()
_ = testKafkaCmd.Process.Kill()
_ = testZkCmd.Process.Kill()
_ = testKafkaCmd.Wait()
_ = testZkCmd.Wait()
_ = os.RemoveAll(testKafkaData)