// SpawnGracefulProducer creates a `KafkaProducer` instance and starts its internal
// goroutines.
func SpawnGracefulProducer(config *config.T) (*GracefulProducer, error) {
saramaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.SaramaConfig())
if err != nil {
return nil, fmt.Errorf("failed to create sarama.Client, err=(%s)", err)
}
saramaProducer, err := sarama.NewAsyncProducerFromClient(saramaClient)
if err != nil {
return nil, fmt.Errorf("failed to create sarama.Producer, err=(%s)", err)
}
gp := &GracefulProducer{
baseCID: sarama.RootCID.NewChild("producer"),
saramaClient: saramaClient,
saramaProducer: saramaProducer,
shutdownTimeout: config.Producer.ShutdownTimeout,
deadMessageCh: config.Producer.DeadMessageCh,
dispatcherCh: make(chan *sarama.ProducerMessage, config.ChannelBufferSize),
resultCh: make(chan produceResult, config.ChannelBufferSize),
}
spawn(&gp.wg, gp.merge)
spawn(&gp.wg, gp.dispatch)
return gp, nil
}
// SpawnSmartConsumer creates a SmartConsumer instance with the specified
// configuration and starts all its goroutines.
func SpawnSmartConsumer(config *config.T) (*SmartConsumer, error) {
kafkaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.SaramaConfig())
if err != nil {
return nil, ErrConsumerSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err))
}
offsetMgr, err := sarama.NewOffsetManagerFromClient(kafkaClient)
if err != nil {
return nil, ErrConsumerSetup(fmt.Errorf("failed to create sarama.OffsetManager: err=(%v)", err))
}
kazooConn, err := kazoo.NewKazoo(config.ZooKeeper.SeedPeers, config.KazooConfig())
if err != nil {
return nil, ErrConsumerSetup(fmt.Errorf("failed to create kazoo.Kazoo: err=(%v)", err))
}
sc := &SmartConsumer{
baseCID: sarama.RootCID.NewChild("smartConsumer"),
config: config,
kafkaClient: kafkaClient,
offsetMgr: offsetMgr,
kazooConn: kazooConn,
}
sc.dispatcher = newDispatcher(sc.baseCID, sc, sc.config)
sc.dispatcher.start()
return sc, nil
}