func offsets(client sarama.Client, topic string, partition int32) (oldest int64, newest int64) {
oldest, err := client.GetOffset(topic, partition, sarama.OffsetOldest)
must(err)
newest, err = client.GetOffset(topic, partition, sarama.OffsetNewest)
must(err)
return oldest, newest
}
func readOffsets(client *sarama.Client, topic string, partitions []int32) {
for _, partition := range partitions {
earliest, err := client.GetOffset(topic, partition, sarama.EarliestOffset)
if err != nil {
logger.Fatalf("error getting earliest offset for %s:%d: %s", topic, partition, err)
}
latest, err := client.GetOffset(topic, partition, sarama.LatestOffsets)
if err != nil {
logger.Fatalf("error getting latest offset for %s:%d: %s", topic, partition, err)
}
logger.Printf("topic=%s partition=%d earliest=%d latest=%d", topic, partition, earliest, latest)
}
}
func readTopic(client sarama.Client, name string) (topic, error) {
t := topic{Name: name}
if config.topic.partitions {
ps, err := client.Partitions(name)
if err != nil {
return t, err
}
for _, p := range ps {
np := partition{Id: p}
oldest, err := client.GetOffset(name, p, sarama.OffsetOldest)
if err != nil {
return t, err
}
np.OldestOffset = oldest
newest, err := client.GetOffset(name, p, sarama.OffsetNewest)
if err != nil {
return t, err
}
np.NewestOffset = newest
if config.topic.leaders {
b, err := client.Leader(name, p)
if err != nil {
return t, err
}
np.Leader = b.Addr()
}
if config.topic.replicas {
rs, err := client.Replicas(name, p)
if err != nil {
return t, err
}
np.Replicas = rs
}
t.Partitions = append(t.Partitions, np)
}
}
return t, nil
}
func (this *Peek) simpleConsumeTopic(zkcluster *zk.ZkCluster, kfk sarama.Client, topic string, partitionId int32,
msgCh chan *sarama.ConsumerMessage) {
consumer, err := sarama.NewConsumerFromClient(kfk)
if err != nil {
panic(err)
}
defer consumer.Close()
if partitionId == -1 {
// all partitions
partitions, err := kfk.Partitions(topic)
if err != nil {
panic(err)
}
for _, p := range partitions {
offset := this.offset
if this.lastN > 0 {
latestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetNewest)
swallow(err)
oldestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetOldest)
swallow(err)
offset = latestOffset - this.lastN
if offset < oldestOffset {
offset = oldestOffset
}
if offset == 0 {
// no message in store
return
}
}
go this.consumePartition(zkcluster, kfk, consumer, topic, p, msgCh, offset)
}
} else {
offset := this.offset
if this.lastN > 0 {
latestOffset, err := kfk.GetOffset(topic, partitionId, sarama.OffsetNewest)
swallow(err)
offset = latestOffset - this.lastN
if offset < 0 {
offset = sarama.OffsetOldest
}
}
this.consumePartition(zkcluster, kfk, consumer, topic, partitionId, msgCh, offset)
}
}
