Exemplo n.º 1
0
func offsets(client sarama.Client, topic string, partition int32) (oldest int64, newest int64) {
	oldest, err := client.GetOffset(topic, partition, sarama.OffsetOldest)
	must(err)
	newest, err = client.GetOffset(topic, partition, sarama.OffsetNewest)
	must(err)
	return oldest, newest
}
Exemplo n.º 2
0
func readOffsets(client *sarama.Client, topic string, partitions []int32) {
	for _, partition := range partitions {
		earliest, err := client.GetOffset(topic, partition, sarama.EarliestOffset)
		if err != nil {
			logger.Fatalf("error getting earliest offset for %s:%d: %s", topic, partition, err)
		}

		latest, err := client.GetOffset(topic, partition, sarama.LatestOffsets)
		if err != nil {
			logger.Fatalf("error getting latest offset for %s:%d: %s", topic, partition, err)
		}

		logger.Printf("topic=%s partition=%d earliest=%d latest=%d", topic, partition, earliest, latest)
	}
}
Exemplo n.º 3
0
Arquivo: topic.go Projeto: fgeller/kt
func readTopic(client sarama.Client, name string) (topic, error) {
	t := topic{Name: name}

	if config.topic.partitions {
		ps, err := client.Partitions(name)
		if err != nil {
			return t, err
		}

		for _, p := range ps {
			np := partition{Id: p}

			oldest, err := client.GetOffset(name, p, sarama.OffsetOldest)
			if err != nil {
				return t, err
			}
			np.OldestOffset = oldest

			newest, err := client.GetOffset(name, p, sarama.OffsetNewest)
			if err != nil {
				return t, err
			}
			np.NewestOffset = newest

			if config.topic.leaders {
				b, err := client.Leader(name, p)
				if err != nil {
					return t, err
				}
				np.Leader = b.Addr()
			}

			if config.topic.replicas {
				rs, err := client.Replicas(name, p)
				if err != nil {
					return t, err
				}
				np.Replicas = rs
			}

			t.Partitions = append(t.Partitions, np)
		}
	}

	return t, nil
}
Exemplo n.º 4
0
func (this *Peek) simpleConsumeTopic(zkcluster *zk.ZkCluster, kfk sarama.Client, topic string, partitionId int32,
	msgCh chan *sarama.ConsumerMessage) {
	consumer, err := sarama.NewConsumerFromClient(kfk)
	if err != nil {
		panic(err)
	}
	defer consumer.Close()

	if partitionId == -1 {
		// all partitions
		partitions, err := kfk.Partitions(topic)
		if err != nil {
			panic(err)
		}

		for _, p := range partitions {
			offset := this.offset
			if this.lastN > 0 {
				latestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, p, sarama.OffsetOldest)
				swallow(err)

				offset = latestOffset - this.lastN
				if offset < oldestOffset {
					offset = oldestOffset
				}

				if offset == 0 {
					// no message in store
					return
				}
			}

			go this.consumePartition(zkcluster, kfk, consumer, topic, p, msgCh, offset)
		}

	} else {
		offset := this.offset
		if this.lastN > 0 {
			latestOffset, err := kfk.GetOffset(topic, partitionId, sarama.OffsetNewest)
			swallow(err)
			offset = latestOffset - this.lastN
			if offset < 0 {
				offset = sarama.OffsetOldest
			}
		}
		this.consumePartition(zkcluster, kfk, consumer, topic, partitionId, msgCh, offset)
	}

}
Exemplo n.º 5
0
func unmountOnInt(c chan os.Signal, server *fuse.Server, client *sarama.Client) {
	s := <-c
	fmt.Println("Got signal:", s)
	server.Unmount()
	client.Close()
}
Exemplo n.º 6
0
		{"N3", []string{"N1", "N2", "N3"}, []int32{0, 1}, []int32{}},
		{"N1", []string{"N1", "N2", "N3"}, []int32{0, 2, 4, 6, 8}, []int32{0, 2}},
		{"N2", []string{"N1", "N2", "N3"}, []int32{0, 2, 4, 6, 8}, []int32{4, 6}},
		{"N3", []string{"N1", "N2", "N3"}, []int32{0, 2, 4, 6, 8}, []int32{8}},
		{"N1", []string{"N1", "N2"}, []int32{0, 1, 2, 3, 4}, []int32{0, 1, 2}},
		{"N9", []string{"N1", "N2"}, []int32{0, 1}, []int32{}},
		{"N1", []string{"N1", "N2", "N3"}, []int32{0}, []int32{0}},
		{"N2", []string{"N1", "N2", "N3"}, []int32{0}, []int32{}},
		{"N3", []string{"N1", "N2", "N3"}, []int32{0}, []int32{}},
		{"N1", []string{"N1", "N2"}, []int32{0, 1, 2, 3, 4}, []int32{0, 1, 2}},
		{"N2", []string{"N1", "N2"}, []int32{0, 1, 2, 3, 4}, []int32{3, 4}},
	}

	Describe("instances", func() {
		var zk *ZK
		var client *sarama.Client
		var subject *ConsumerGroup
		var err error
		var consumerConfig = &sarama.ConsumerConfig{MaxWaitTime: 200, EventBufferSize: 10}

		var runConsumerCycle = func(errors chan error, events chan int64, n int) {
			group, err := NewConsumerGroup(client, zk, tnG, tnT, nil, consumerConfig)
			if err != nil {
				errors <- err
				return
			}
			defer group.Close()

			for i := 0; i < n; i++ {
				group.Process(func(b *EventBatch) error {
					for _, e := range b.Events {
Exemplo n.º 7
0
	"testing"

	"github.com/Shopify/sarama"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	testGroup     = "sarama-cluster-group"
	testKafkaData = "/tmp/sarama-cluster-test"
)

var (
	testKafkaRoot  = "kafka_2.11-0.9.0.0"
	testKafkaAddrs = []string{"127.0.0.1:29092"}
	testTopics     = []string{"topic-a", "topic-b"}

	testClient              sarama.Client
	testKafkaCmd, testZkCmd *exec.Cmd
)

func init() {
	if dir := os.Getenv("KAFKA_DIR"); dir != "" {
		testKafkaRoot = dir
	}
}

var _ = Describe("offsetInfo", func() {

	It("should calculate next offset", func() {
		Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(sarama.OffsetOldest))
		Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetNewest)).To(Equal(sarama.OffsetNewest))