Пример #1
0
func (this *Peek) simpleConsumeTopic(zkcluster *zk.ZkCluster, kfk sarama.Client, topic string, partitionId int32,
	msgCh chan *sarama.ConsumerMessage) {
	consumer, err := sarama.NewConsumerFromClient(kfk)
	if err != nil {
		panic(err)
	}
	defer consumer.Close()

	if partitionId == -1 {
		// all partitions
		partitions, err := kfk.Partitions(topic)
		if err != nil {
			panic(err)
		}

		for _, p := range partitions {
			offset := this.offset
			if this.lastN > 0 {
				latestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetOldest)
				swallow(err)

				offset = latestOffset - this.lastN
				if offset < oldestOffset {
					offset = oldestOffset
				}

				if offset == 0 {
					// no message in store
					return
				}
			}

			go this.consumePartition(zkcluster, kfk, consumer, topic, p, msgCh, offset)
		}

	} else {
		offset := this.offset
		if this.lastN > 0 {
			latestOffset, err := kfk.GetOffset(topic, partitionId, sarama.OffsetNewest)
			swallow(err)
			offset = latestOffset - this.lastN
			if offset < 0 {
				offset = sarama.OffsetOldest
			}
		}
		this.consumePartition(zkcluster, kfk, consumer, topic, partitionId, msgCh, offset)
	}

}
Пример #2
0
func readTopic(client sarama.Client, name string) (topic, error) {
	t := topic{Name: name}

	if config.topic.partitions {
		ps, err := client.Partitions(name)
		if err != nil {
			return t, err
		}

		for _, p := range ps {
			np := partition{Id: p}

			oldest, err := client.GetOffset(name, p, sarama.OffsetOldest)
			if err != nil {
				return t, err
			}
			np.OldestOffset = oldest

			newest, err := client.GetOffset(name, p, sarama.OffsetNewest)
			if err != nil {
				return t, err
			}
			np.NewestOffset = newest

			if config.topic.leaders {
				b, err := client.Leader(name, p)
				if err != nil {
					return t, err
				}
				np.Leader = b.Addr()
			}

			if config.topic.replicas {
				rs, err := client.Replicas(name, p)
				if err != nil {
					return t, err
				}
				np.Replicas = rs
			}

			t.Partitions = append(t.Partitions, np)
		}
	}

	return t, nil
}
Пример #3
0
	Expect(os.MkdirAll(testKafkaData, 0777)).NotTo(HaveOccurred())
	Expect(testZkCmd.Start()).NotTo(HaveOccurred())
	Expect(testKafkaCmd.Start()).NotTo(HaveOccurred())

	// Wait for client
	Eventually(func() error {
		var err error

		testClient, err = sarama.NewClient(testKafkaAddrs, nil)
		return err
	}, "10s", "1s").ShouldNot(HaveOccurred())

	// Ensure we can retrieve partition info
	Eventually(func() error {
		_, err := testClient.Partitions(testTopics[0])
		return err
	}, "10s", "500ms").ShouldNot(HaveOccurred())

	// Seed a few messages
	Expect(testSeed(1000)).NotTo(HaveOccurred())
})

var _ = AfterSuite(func() {
	_ = testClient.Close()

	_ = testKafkaCmd.Process.Kill()
	_ = testZkCmd.Process.Kill()
	_ = testKafkaCmd.Wait()
	_ = testZkCmd.Wait()
	_ = os.RemoveAll(testKafkaData)