func ExampleBroker_Consumer() {
broker := NewBroker()
msg := &proto.Message{Value: []byte("first")}
// mock server actions, pushing data through consumer
go func() {
consumer, _ := broker.Consumer(kafka.NewConsumerConf("my-topic", 0))
c := consumer.(*Consumer)
// it is possible to send messages through consumer...
c.Messages <- msg
// every consumer fetch call is blocking untill there is either message
// or error ready to return, this way we can test slow consumers
time.Sleep(time.Millisecond * 20)
// ...as well as push errors to mock failure
c.Errors <- errors.New("expected error is expected")
}()
// test broker never fails creating consumer
consumer, _ := broker.Consumer(kafka.NewConsumerConf("my-topic", 0))
m, err := consumer.Consume()
if err == nil {
fmt.Printf("Value: %q\n", m.Value)
}
if _, err = consumer.Consume(); err != nil {
fmt.Printf("Error: %s\n", err)
}
// output:
//
// Value: "first"
// Error: expected error is expected
}
func TestNewTopic(t *testing.T) {
IntegrationTest(t)
const msgPerTopic = 10
topic := "NewTopic"
cluster := NewKafkaCluster("kafka-docker/", 1)
if err := cluster.Start(); err != nil {
t.Fatalf("cannot start kafka cluster: %s", err)
}
defer func() {
_ = cluster.Stop()
}()
bconf := kafka.NewBrokerConf("producer-new-topic")
bconf.Logger = &testLogger{t}
bconf.AllowTopicCreation = true
addrs, err := cluster.KafkaAddrs()
if err != nil {
t.Fatalf("cannot get kafka address: %s", err)
}
broker, err := kafka.Dial(addrs, bconf)
if err != nil {
t.Fatalf("cannot connect to cluster (%q): %s", addrs, err)
}
defer broker.Close()
m := proto.Message{
Value: []byte("Test message"),
}
pconf := kafka.NewProducerConf()
producer := broker.Producer(pconf)
if _, err := producer.Produce(topic, 0, &m); err != nil {
t.Fatalf("cannot produce to %q: %s", topic, err)
}
consumer, err := broker.Consumer(kafka.NewConsumerConf(topic, 0))
if err != nil {
t.Fatalf("cannot create consumer for %q: %s", topic, err)
}
if _, err := consumer.Consume(); err != nil {
t.Errorf("cannot consume message from %q: %s", topic, err)
}
}
// printConsumed read messages from kafka and print them out
func printConsumed(broker kafka.Client) {
conf := kafka.NewConsumerConf(topic, partition)
conf.StartOffset = kafka.StartOffsetNewest
consumer, err := broker.Consumer(conf)
if err != nil {
log.Fatalf("cannot create kafka consumer for %s:%d: %s", topic, partition, err)
}
for {
msg, err := consumer.Consume()
if err != nil {
if err != kafka.ErrNoData {
log.Printf("cannot consume %q topic message: %s", topic, err)
}
break
}
log.Printf("message %d: %s", msg.Offset, msg.Value)
}
log.Print("consumer quit")
}
func (self *KafkaInput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) {
log.Println("KafkaInput Init.")
self.common = pcf
hn, err := os.Hostname()
if err != nil {
hn = "kamanclient"
}
self.config = &KafkaInputConfig{
ClientId: hn,
Partitions: 0,
FlushInterval: 1000,
}
if err = toml.PrimitiveDecode(conf, self.config); err != nil {
return fmt.Errorf("Can't unmarshal KafkaInput config: %s", err)
}
if len(self.config.Addrs) == 0 {
return errors.New("addrs must have at least one entry")
}
if len(self.config.Topic) == 0 {
return fmt.Errorf("topic is empty")
}
bcf := kafka.NewBrokerConf(self.config.ClientId)
bcf.AllowTopicCreation = false
//bcf.Logger = &stdLogger{}
self.broker, err = kafka.Dial(self.config.Addrs, bcf)
if err != nil {
return fmt.Errorf("cannot connect to kafka cluster: %s", err)
}
defer self.broker.Close()
consumerconf := kafka.NewConsumerConf(self.config.Topic, self.config.Partition)
self.consumer, err = self.broker.Consumer(consumerconf)
if err != nil {
return fmt.Errorf("cannot create kafka consumer for %s:%d: %s", self.config.Topic, self.config.Partition, err)
}
return err
}
func TestConsumerBrokenConnection(t *testing.T) {
IntegrationTest(t)
const msgPerTopic = 10
topics := []string{"Topic3", "Topic4"}
cluster := NewKafkaCluster("kafka-docker/", 4)
if err := cluster.Start(); err != nil {
t.Fatalf("cannot start kafka cluster: %s", err)
}
defer func() {
_ = cluster.Stop()
}()
bconf := kafka.NewBrokerConf("producer-broken-connection")
bconf.Logger = &testLogger{t}
addrs, err := cluster.KafkaAddrs()
if err != nil {
t.Fatalf("cannot get kafka address: %s", err)
}
broker, err := kafka.Dial(addrs, bconf)
if err != nil {
t.Fatalf("cannot connect to cluster (%q): %s", addrs, err)
}
defer broker.Close()
// produce big message to enforce TCP buffer flush
m := proto.Message{
Value: []byte(strings.Repeat("consumer broken connection message ", 1000)),
}
pconf := kafka.NewProducerConf()
producer := broker.Producer(pconf)
// send message to all topics
for _, name := range topics {
for i := 0; i < msgPerTopic; i++ {
if _, err := producer.Produce(name, 0, &m); err != nil {
t.Fatalf("cannot produce to %q: %s", name, err)
}
}
}
// close two kafka clusters and publish to all 3 topics - 2 of them should
// retry sending, because lack of leader makes the request fail
//
// request should not succeed until nodes are back - bring them back after
// small delay and make sure producing was successful
containers, err := cluster.Containers()
if err != nil {
t.Fatalf("cannot get containers: %s", err)
}
var stopped []*Container
for _, container := range containers {
if container.RunningKafka() {
if err := container.Kill(); err != nil {
t.Fatalf("cannot kill %q kafka container: %s", container.ID, err)
}
stopped = append(stopped, container)
}
if len(stopped) == 2 {
break
}
}
// bring stopped containers back
errc := make(chan error)
go func() {
time.Sleep(500 * time.Millisecond)
for _, container := range stopped {
if err := container.Start(); err != nil {
errc <- err
}
}
close(errc)
}()
// make sure data was persisted
for _, name := range topics {
consumer, err := broker.Consumer(kafka.NewConsumerConf(name, 0))
if err != nil {
t.Errorf("cannot create consumer for %q: %s", name, err)
continue
}
for i := 0; i < msgPerTopic; i++ {
if _, err := consumer.Consume(); err != nil {
t.Errorf("cannot consume %d message from %q: %s", i, name, err)
}
}
}
for err := range errc {
t.Errorf("cannot start container: %s", err)
}
}
// kafkaConsumerChannel creates a consumer that continuously attempts to consume messages from
// Kafka for the given partition.
func (w *Marshaler) kafkaConsumerChannel(partID int) <-chan message {
log.Debugf("rationalize[%d]: starting", partID)
out := make(chan message, 1000)
go func() {
var err error
var alive bool
var offsetFirst, offsetNext int64
retry := &backoff.Backoff{Min: 500 * time.Millisecond, Jitter: true}
for ; true; time.Sleep(retry.Duration()) {
// Figure out how many messages are in this topic. This can fail if the broker handling
// this partition is down, so we will loop.
offsetFirst, err = w.kafka.OffsetEarliest(MarshalTopic, int32(partID))
if err != nil {
log.Errorf("rationalize[%d]: failed to get offset: %s", partID, err)
continue
}
offsetNext, err = w.kafka.OffsetLatest(MarshalTopic, int32(partID))
if err != nil {
log.Errorf("rationalize[%d]: failed to get offset: %s", partID, err)
continue
}
log.Debugf("rationalize[%d]: offsets %d to %d", partID, offsetFirst, offsetNext)
// TODO: Is there a case where the latest offset is X>0 but there is no data in
// the partition? does the offset reset to 0?
if offsetNext == 0 || offsetFirst == offsetNext {
alive = true
w.rationalizers.Done()
}
break
}
retry.Reset()
// TODO: Technically we don't have to start at the beginning, we just need to start back
// a couple heartbeat intervals to get a full state of the world. But this is easiest
// for right now...
// TODO: Think about the above. Is it actually true?
consumerConf := kafka.NewConsumerConf(MarshalTopic, int32(partID))
consumerConf.StartOffset = kafka.StartOffsetOldest
consumerConf.RequestTimeout = 1 * time.Second
consumer, err := w.kafka.Consumer(consumerConf)
if err != nil {
// Unfortunately this is a fatal error, as without being able to consume this partition
// we can't effectively rationalize.
log.Fatalf("rationalize[%d]: Failed to create consumer: %s", partID, err)
}
// Consume messages forever, or until told to quit.
for {
if atomic.LoadInt32(w.quit) == 1 {
log.Debugf("rationalize[%d]: terminating.", partID)
close(out)
return
}
msgb, err := consumer.Consume()
if err != nil {
// The internal consumer will do a number of retries. If we get an error here,
// we're probably in the middle of a partition handoff. We should pause so we
// don't hammer the cluster, but otherwise continue.
log.Warningf("rationalize[%d]: failed to consume: %s", partID, err)
time.Sleep(retry.Duration())
continue
}
retry.Reset()
msg, err := decode(msgb.Value)
if err != nil {
// Invalid message in the stream. This should never happen, but if it does, just
// continue on.
// TODO: We should probably think about this. If we end up in a situation where
// one version of this software has a bug that writes invalid messages, it could
// be doing things we don't anticipate. Of course, crashing all consumers
// reading that partition is also bad.
log.Errorf("rationalize[%d]: %s", partID, err)
continue
}
log.Debugf("rationalize[%d]: @%d: [%s]", partID, msgb.Offset, msg.Encode())
out <- msg
// This is a one-time thing that fires the first time the rationalizer comes up
// and makes sure we actually process all of the messages.
if !alive && msgb.Offset >= offsetNext-1 {
for len(out) > 0 {
time.Sleep(100 * time.Millisecond)
}
log.Infof("rationalize[%d]: reached offset %d, now alive",
partID, msgb.Offset)
alive = true
w.rationalizers.Done()
}
}
}()
return out
}