func offsets(client sarama.Client, topic string, partition int32) (oldest int64, newest int64) {
oldest, err := client.GetOffset(topic, partition, sarama.OffsetOldest)
must(err)
newest, err = client.GetOffset(topic, partition, sarama.OffsetNewest)
must(err)
return oldest, newest
}
func readOffsets(client *sarama.Client, topic string, partitions []int32) {
for _, partition := range partitions {
earliest, err := client.GetOffset(topic, partition, sarama.EarliestOffset)
if err != nil {
logger.Fatalf("error getting earliest offset for %s:%d: %s", topic, partition, err)
}
latest, err := client.GetOffset(topic, partition, sarama.LatestOffsets)
if err != nil {
logger.Fatalf("error getting latest offset for %s:%d: %s", topic, partition, err)
}
logger.Printf("topic=%s partition=%d earliest=%d latest=%d", topic, partition, earliest, latest)
}
}
func readTopic(client sarama.Client, name string) (topic, error) {
t := topic{Name: name}
if config.topic.partitions {
ps, err := client.Partitions(name)
if err != nil {
return t, err
}
for _, p := range ps {
np := partition{Id: p}
oldest, err := client.GetOffset(name, p, sarama.OffsetOldest)
if err != nil {
return t, err
}
np.OldestOffset = oldest
newest, err := client.GetOffset(name, p, sarama.OffsetNewest)
if err != nil {
return t, err
}
np.NewestOffset = newest
if config.topic.leaders {
b, err := client.Leader(name, p)
if err != nil {
return t, err
}
np.Leader = b.Addr()
}
if config.topic.replicas {
rs, err := client.Replicas(name, p)
if err != nil {
return t, err
}
np.Replicas = rs
}
t.Partitions = append(t.Partitions, np)
}
}
return t, nil
}
func (this *Peek) simpleConsumeTopic(zkcluster *zk.ZkCluster, kfk sarama.Client, topic string, partitionId int32,
msgCh chan *sarama.ConsumerMessage) {
consumer, err := sarama.NewConsumerFromClient(kfk)
if err != nil {
panic(err)
}
defer consumer.Close()
if partitionId == -1 {
// all partitions
partitions, err := kfk.Partitions(topic)
if err != nil {
panic(err)
}
for _, p := range partitions {
offset := this.offset
if this.lastN > 0 {
latestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetNewest)
swallow(err)
oldestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetOldest)
swallow(err)
offset = latestOffset - this.lastN
if offset < oldestOffset {
offset = oldestOffset
}
if offset == 0 {
// no message in store
return
}
}
go this.consumePartition(zkcluster, kfk, consumer, topic, p, msgCh, offset)
}
} else {
offset := this.offset
if this.lastN > 0 {
latestOffset, err := kfk.GetOffset(topic, partitionId, sarama.OffsetNewest)
swallow(err)
offset = latestOffset - this.lastN
if offset < 0 {
offset = sarama.OffsetOldest
}
}
this.consumePartition(zkcluster, kfk, consumer, topic, partitionId, msgCh, offset)
}
}
func unmountOnInt(c chan os.Signal, server *fuse.Server, client *sarama.Client) {
s := <-c
fmt.Println("Got signal:", s)
server.Unmount()
client.Close()
}
{"N3", []string{"N1", "N2", "N3"}, []int32{0, 1}, []int32{}},
{"N1", []string{"N1", "N2", "N3"}, []int32{0, 2, 4, 6, 8}, []int32{0, 2}},
{"N2", []string{"N1", "N2", "N3"}, []int32{0, 2, 4, 6, 8}, []int32{4, 6}},
{"N3", []string{"N1", "N2", "N3"}, []int32{0, 2, 4, 6, 8}, []int32{8}},
{"N1", []string{"N1", "N2"}, []int32{0, 1, 2, 3, 4}, []int32{0, 1, 2}},
{"N9", []string{"N1", "N2"}, []int32{0, 1}, []int32{}},
{"N1", []string{"N1", "N2", "N3"}, []int32{0}, []int32{0}},
{"N2", []string{"N1", "N2", "N3"}, []int32{0}, []int32{}},
{"N3", []string{"N1", "N2", "N3"}, []int32{0}, []int32{}},
{"N1", []string{"N1", "N2"}, []int32{0, 1, 2, 3, 4}, []int32{0, 1, 2}},
{"N2", []string{"N1", "N2"}, []int32{0, 1, 2, 3, 4}, []int32{3, 4}},
}
Describe("instances", func() {
var zk *ZK
var client *sarama.Client
var subject *ConsumerGroup
var err error
var consumerConfig = &sarama.ConsumerConfig{MaxWaitTime: 200, EventBufferSize: 10}
var runConsumerCycle = func(errors chan error, events chan int64, n int) {
group, err := NewConsumerGroup(client, zk, tnG, tnT, nil, consumerConfig)
if err != nil {
errors <- err
return
}
defer group.Close()
for i := 0; i < n; i++ {
group.Process(func(b *EventBatch) error {
for _, e := range b.Events {
"testing"
"github.com/Shopify/sarama"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
testGroup = "sarama-cluster-group"
testKafkaData = "/tmp/sarama-cluster-test"
)
var (
testKafkaRoot = "kafka_2.11-0.9.0.0"
testKafkaAddrs = []string{"127.0.0.1:29092"}
testTopics = []string{"topic-a", "topic-b"}
testClient sarama.Client
testKafkaCmd, testZkCmd *exec.Cmd
)
func init() {
if dir := os.Getenv("KAFKA_DIR"); dir != "" {
testKafkaRoot = dir
}
}
var _ = Describe("offsetInfo", func() {
It("should calculate next offset", func() {
Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(sarama.OffsetOldest))
Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetNewest)).To(Equal(sarama.OffsetNewest))