func ExampleBroker_Producer() { broker := NewBroker() msg := &proto.Message{Value: []byte("first")} producer := broker.Producer(kafka.NewProducerConf()) // mock server actions, handling any produce call go func() { resp, err := broker.ReadProducers(time.Millisecond * 20) if err != nil { panic(fmt.Sprintf("failed reading producers: %s", err)) } if len(resp.Messages) != 1 { panic("expected single message") } if !reflect.DeepEqual(resp.Messages[0], msg) { panic("expected different message") } }() // provide data for above goroutine _, err := producer.Produce("my-topic", 0, msg) if err != nil { panic(fmt.Sprintf("cannot produce message: %s", err)) } mockProducer := producer.(*Producer) // test error handling by forcing producer to return error, // // it is possible to manipulate produce result by changing producer's // ResponseOffset and ResponseError attributes mockProducer.ResponseError = errors.New("my spoon is too big!") _, err = producer.Produce("my-topic", 0, msg) fmt.Printf("Error: %s\n", err) // output: // // Error: my spoon is too big! }
func TestProducerBrokenConnection(t *testing.T) { IntegrationTest(t) topics := []string{"Topic3", "Topic4"} cluster := NewKafkaCluster("kafka-docker/", 4) if err := cluster.Start(); err != nil { t.Fatalf("cannot start kafka cluster: %s", err) } defer func() { _ = cluster.Stop() }() bconf := kafka.NewBrokerConf("producer-broken-connection") addrs, err := cluster.KafkaAddrs() if err != nil { t.Fatalf("cannot get kafka address: %s", err) } broker, err := kafka.Dial(addrs, bconf) if err != nil { t.Fatalf("cannot connect to cluster (%q): %s", addrs, err) } defer broker.Close() // produce big message to enforce TCP buffer flush m := proto.Message{ Value: []byte(strings.Repeat("producer broken connection message ", 1000)), } pconf := kafka.NewProducerConf() producer := broker.Producer(pconf) // send message to all topics to make sure it's working for _, name := range topics { if _, err := producer.Produce(name, 0, &m); err != nil { t.Fatalf("cannot produce to %q: %s", name, err) } } // close two kafka clusters and publish to all 3 topics - 2 of them should // retry sending, because lack of leader makes the request fail // // request should not succeed until nodes are back - bring them back after // small delay and make sure producing was successful containers, err := cluster.Containers() if err != nil { t.Fatalf("cannot get containers: %s", err) } var stopped []*Container for _, container := range containers { if container.RunningKafka() { if err := container.Kill(); err != nil { t.Fatalf("cannot kill %q kafka container: %s", container.ID, err) } stopped = append(stopped, container) } if len(stopped) == 2 { break } } // bring stopped containers back errc := make(chan error) go func() { time.Sleep(500 * time.Millisecond) for _, container := range stopped { if err := container.Start(); err != nil { errc <- err } } close(errc) }() // send message to all topics to make sure it's working for _, name := range topics { if _, err := producer.Produce(name, 0, &m); err != nil { t.Errorf("cannot produce to %q: %s", name, err) } } for err := range errc { t.Errorf("cannot start container: %s", err) } // make sure data was persisted for _, name := range topics { consumer, err := broker.Consumer(kafka.NewConsumerConf(name, 0)) if err != nil { t.Errorf("cannot create consumer for %q: %s", name, err) continue } for i := 0; i < 2; i++ { if _, err := consumer.Consume(); err != nil { t.Errorf("cannot consume %d message from %q: %s", i, name, err) } } } }
// Dial returns a new cluster object which can be used to instantiate a number of Marshalers // that all use the same cluster. You may pass brokerConf or may set it to nil. func Dial(name string, brokers []string, options MarshalOptions) (*KafkaCluster, error) { // Connect to Kafka brokerConf := kafka.NewBrokerConf("PortalMarshal") brokerConf.MetadataRefreshFrequency = time.Hour brokerConf.ConnectionLimit = options.BrokerConnectionLimit brokerConf.LeaderRetryLimit = 1 // Do not retry broker, err := kafka.Dial(brokers, brokerConf) if err != nil { return nil, err } c := &KafkaCluster{ quit: new(int32), rsteps: new(int32), name: name, options: options, lock: &sync.RWMutex{}, rationalizers: &sync.WaitGroup{}, broker: broker, producer: broker.Producer(kafka.NewProducerConf()), topics: make(map[string]int), groups: make(map[string]map[string]*topicState), pausedGroups: make(map[string]time.Time), jitters: make(chan time.Duration, 100), } // Do an initial metadata fetch, this will block a bit err = c.refreshMetadata() if err != nil { return nil, fmt.Errorf("Failed to get metadata: %s", err) } // If there is no marshal topic, then we can't run. The admins must go create the topic // before they can use this library. Please see the README. c.partitions = c.getTopicPartitions(MarshalTopic) if c.partitions == 0 { return nil, errors.New("Marshalling topic not found. Please see the documentation.") } // Now we start a goroutine to start consuming each of the partitions in the marshal // topic. Note that this doesn't handle increasing the partition count on that topic // without stopping all consumers. c.rationalizers.Add(c.partitions) for id := 0; id < c.partitions; id++ { go c.rationalize(id, c.kafkaConsumerChannel(id)) } // A jitter calculator, just fills a channel with random numbers so that other // people don't have to build their own random generator. It is important that // these values be somewhat less than the HeartbeatInterval as we use this for // jittering our heartbeats. go func() { rnd := rand.New(rand.NewSource(time.Now().UnixNano())) for { jitter := rnd.Intn(HeartbeatInterval/2) + (HeartbeatInterval / 4) c.jitters <- time.Duration(jitter) * time.Second } }() // Now start the metadata refreshing goroutine go func() { for !c.Terminated() { time.Sleep(<-c.jitters) log.Infof("[%s] Refreshing topic metadata.", c.name) c.refreshMetadata() // See if the number of partitions in the marshal topic changed. This is bad if // it happens, since it means we can no longer coordinate correctly. if c.getTopicPartitions(MarshalTopic) != c.partitions { log.Errorf("[%s] Marshal topic partition count changed. Terminating!", c.name) c.Terminate() } } }() // Wait for all rationalizers to come alive log.Infof("[%s] Waiting for all rationalizers to come alive.", c.name) c.rationalizers.Wait() log.Infof("[%s] All rationalizers alive, KafkaCluster now alive.", c.name) return c, nil }