Example #1
0
func ExampleBroker_Consumer() {
	broker := NewBroker()
	msg := &proto.Message{Value: []byte("first")}

	// mock server actions, pushing data through consumer
	go func() {
		consumer, _ := broker.Consumer(kafka.NewConsumerConf("my-topic", 0))
		c := consumer.(*Consumer)
		// it is possible to send messages through consumer...
		c.Messages <- msg

		// every consumer fetch call is blocking untill there is either message
		// or error ready to return, this way we can test slow consumers
		time.Sleep(time.Millisecond * 20)

		// ...as well as push errors to mock failure
		c.Errors <- errors.New("expected error is expected")
	}()

	// test broker never fails creating consumer
	consumer, _ := broker.Consumer(kafka.NewConsumerConf("my-topic", 0))

	m, err := consumer.Consume()
	if err == nil {
		fmt.Printf("Value: %q\n", m.Value)
	}
	if _, err = consumer.Consume(); err != nil {
		fmt.Printf("Error: %s\n", err)
	}

	// output:
	//
	// Value: "first"
	// Error: expected error is expected
}
Example #2
0
func TestNewTopic(t *testing.T) {
	IntegrationTest(t)
	const msgPerTopic = 10

	topic := "NewTopic"

	cluster := NewKafkaCluster("kafka-docker/", 1)
	if err := cluster.Start(); err != nil {
		t.Fatalf("cannot start kafka cluster: %s", err)
	}
	defer func() {
		_ = cluster.Stop()
	}()

	bconf := kafka.NewBrokerConf("producer-new-topic")
	bconf.Logger = &testLogger{t}
	bconf.AllowTopicCreation = true
	addrs, err := cluster.KafkaAddrs()
	if err != nil {
		t.Fatalf("cannot get kafka address: %s", err)
	}
	broker, err := kafka.Dial(addrs, bconf)
	if err != nil {
		t.Fatalf("cannot connect to cluster (%q): %s", addrs, err)
	}
	defer broker.Close()

	m := proto.Message{
		Value: []byte("Test message"),
	}
	pconf := kafka.NewProducerConf()
	producer := broker.Producer(pconf)

	if _, err := producer.Produce(topic, 0, &m); err != nil {
		t.Fatalf("cannot produce to %q: %s", topic, err)
	}

	consumer, err := broker.Consumer(kafka.NewConsumerConf(topic, 0))
	if err != nil {
		t.Fatalf("cannot create consumer for %q: %s", topic, err)
	}
	if _, err := consumer.Consume(); err != nil {
		t.Errorf("cannot consume message from %q: %s", topic, err)
	}
}
Example #3
0
// printConsumed read messages from kafka and print them out
func printConsumed(broker kafka.Client) {
	conf := kafka.NewConsumerConf(topic, partition)
	conf.StartOffset = kafka.StartOffsetNewest
	consumer, err := broker.Consumer(conf)
	if err != nil {
		log.Fatalf("cannot create kafka consumer for %s:%d: %s", topic, partition, err)
	}

	for {
		msg, err := consumer.Consume()
		if err != nil {
			if err != kafka.ErrNoData {
				log.Printf("cannot consume %q topic message: %s", topic, err)
			}
			break
		}
		log.Printf("message %d: %s", msg.Offset, msg.Value)
	}
	log.Print("consumer quit")
}
Example #4
0
func (self *KafkaInput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) {
	log.Println("KafkaInput Init.")
	self.common = pcf
	hn, err := os.Hostname()
	if err != nil {
		hn = "kamanclient"
	}
	self.config = &KafkaInputConfig{
		ClientId:      hn,
		Partitions:    0,
		FlushInterval: 1000,
	}
	if err = toml.PrimitiveDecode(conf, self.config); err != nil {
		return fmt.Errorf("Can't unmarshal KafkaInput config: %s", err)
	}
	if len(self.config.Addrs) == 0 {
		return errors.New("addrs must have at least one entry")
	}
	if len(self.config.Topic) == 0 {
		return fmt.Errorf("topic is empty")
	}

	bcf := kafka.NewBrokerConf(self.config.ClientId)
	bcf.AllowTopicCreation = false
	//bcf.Logger = &stdLogger{}

	self.broker, err = kafka.Dial(self.config.Addrs, bcf)
	if err != nil {
		return fmt.Errorf("cannot connect to kafka cluster: %s", err)
	}

	defer self.broker.Close()
	consumerconf := kafka.NewConsumerConf(self.config.Topic, self.config.Partition)
	self.consumer, err = self.broker.Consumer(consumerconf)
	if err != nil {
		return fmt.Errorf("cannot create kafka consumer for %s:%d: %s", self.config.Topic, self.config.Partition, err)
	}
	return err
}
Example #5
0
func TestConsumerBrokenConnection(t *testing.T) {
	IntegrationTest(t)
	const msgPerTopic = 10

	topics := []string{"Topic3", "Topic4"}

	cluster := NewKafkaCluster("kafka-docker/", 4)
	if err := cluster.Start(); err != nil {
		t.Fatalf("cannot start kafka cluster: %s", err)
	}
	defer func() {
		_ = cluster.Stop()
	}()

	bconf := kafka.NewBrokerConf("producer-broken-connection")
	bconf.Logger = &testLogger{t}
	addrs, err := cluster.KafkaAddrs()
	if err != nil {
		t.Fatalf("cannot get kafka address: %s", err)
	}
	broker, err := kafka.Dial(addrs, bconf)
	if err != nil {
		t.Fatalf("cannot connect to cluster (%q): %s", addrs, err)
	}
	defer broker.Close()

	// produce big message to enforce TCP buffer flush
	m := proto.Message{
		Value: []byte(strings.Repeat("consumer broken connection message ", 1000)),
	}
	pconf := kafka.NewProducerConf()
	producer := broker.Producer(pconf)

	// send message to all topics
	for _, name := range topics {
		for i := 0; i < msgPerTopic; i++ {
			if _, err := producer.Produce(name, 0, &m); err != nil {
				t.Fatalf("cannot produce to %q: %s", name, err)
			}
		}
	}

	// close two kafka clusters and publish to all 3 topics - 2 of them should
	// retry sending, because lack of leader makes the request fail
	//
	// request should not succeed until nodes are back - bring them back after
	// small delay and make sure producing was successful
	containers, err := cluster.Containers()
	if err != nil {
		t.Fatalf("cannot get containers: %s", err)
	}
	var stopped []*Container
	for _, container := range containers {
		if container.RunningKafka() {
			if err := container.Kill(); err != nil {
				t.Fatalf("cannot kill %q kafka container: %s", container.ID, err)
			}
			stopped = append(stopped, container)
		}
		if len(stopped) == 2 {
			break
		}
	}

	// bring stopped containers back
	errc := make(chan error)
	go func() {
		time.Sleep(500 * time.Millisecond)
		for _, container := range stopped {
			if err := container.Start(); err != nil {
				errc <- err
			}
		}
		close(errc)
	}()

	// make sure data was persisted
	for _, name := range topics {
		consumer, err := broker.Consumer(kafka.NewConsumerConf(name, 0))
		if err != nil {
			t.Errorf("cannot create consumer for %q: %s", name, err)
			continue
		}
		for i := 0; i < msgPerTopic; i++ {
			if _, err := consumer.Consume(); err != nil {
				t.Errorf("cannot consume %d message from %q: %s", i, name, err)
			}
		}
	}

	for err := range errc {
		t.Errorf("cannot start container: %s", err)
	}
}
Example #6
0
// kafkaConsumerChannel creates a consumer that continuously attempts to consume messages from
// Kafka for the given partition.
func (w *Marshaler) kafkaConsumerChannel(partID int) <-chan message {
	log.Debugf("rationalize[%d]: starting", partID)
	out := make(chan message, 1000)
	go func() {
		var err error
		var alive bool
		var offsetFirst, offsetNext int64

		retry := &backoff.Backoff{Min: 500 * time.Millisecond, Jitter: true}
		for ; true; time.Sleep(retry.Duration()) {
			// Figure out how many messages are in this topic. This can fail if the broker handling
			// this partition is down, so we will loop.
			offsetFirst, err = w.kafka.OffsetEarliest(MarshalTopic, int32(partID))
			if err != nil {
				log.Errorf("rationalize[%d]: failed to get offset: %s", partID, err)
				continue
			}
			offsetNext, err = w.kafka.OffsetLatest(MarshalTopic, int32(partID))
			if err != nil {
				log.Errorf("rationalize[%d]: failed to get offset: %s", partID, err)
				continue
			}
			log.Debugf("rationalize[%d]: offsets %d to %d", partID, offsetFirst, offsetNext)

			// TODO: Is there a case where the latest offset is X>0 but there is no data in
			// the partition? does the offset reset to 0?
			if offsetNext == 0 || offsetFirst == offsetNext {
				alive = true
				w.rationalizers.Done()
			}
			break
		}
		retry.Reset()

		// TODO: Technically we don't have to start at the beginning, we just need to start back
		// a couple heartbeat intervals to get a full state of the world. But this is easiest
		// for right now...
		// TODO: Think about the above. Is it actually true?
		consumerConf := kafka.NewConsumerConf(MarshalTopic, int32(partID))
		consumerConf.StartOffset = kafka.StartOffsetOldest
		consumerConf.RequestTimeout = 1 * time.Second

		consumer, err := w.kafka.Consumer(consumerConf)
		if err != nil {
			// Unfortunately this is a fatal error, as without being able to consume this partition
			// we can't effectively rationalize.
			log.Fatalf("rationalize[%d]: Failed to create consumer: %s", partID, err)
		}

		// Consume messages forever, or until told to quit.
		for {
			if atomic.LoadInt32(w.quit) == 1 {
				log.Debugf("rationalize[%d]: terminating.", partID)
				close(out)
				return
			}

			msgb, err := consumer.Consume()
			if err != nil {
				// The internal consumer will do a number of retries. If we get an error here,
				// we're probably in the middle of a partition handoff. We should pause so we
				// don't hammer the cluster, but otherwise continue.
				log.Warningf("rationalize[%d]: failed to consume: %s", partID, err)
				time.Sleep(retry.Duration())
				continue
			}
			retry.Reset()

			msg, err := decode(msgb.Value)
			if err != nil {
				// Invalid message in the stream. This should never happen, but if it does, just
				// continue on.
				// TODO: We should probably think about this. If we end up in a situation where
				// one version of this software has a bug that writes invalid messages, it could
				// be doing things we don't anticipate. Of course, crashing all consumers
				// reading that partition is also bad.
				log.Errorf("rationalize[%d]: %s", partID, err)
				continue
			}

			log.Debugf("rationalize[%d]: @%d: [%s]", partID, msgb.Offset, msg.Encode())
			out <- msg

			// This is a one-time thing that fires the first time the rationalizer comes up
			// and makes sure we actually process all of the messages.
			if !alive && msgb.Offset >= offsetNext-1 {
				for len(out) > 0 {
					time.Sleep(100 * time.Millisecond)
				}
				log.Infof("rationalize[%d]: reached offset %d, now alive",
					partID, msgb.Offset)
				alive = true
				w.rationalizers.Done()
			}
		}
	}()
	return out
}