// Spawn creates a producer instance and starts its internal goroutines.
func Spawn(cfg *config.T) (*T, error) {
saramaCfg := sarama.NewConfig()
saramaCfg.ChannelBufferSize = cfg.Producer.ChannelBufferSize
saramaCfg.Producer.RequiredAcks = sarama.WaitForAll
saramaCfg.Producer.Return.Successes = true
saramaCfg.Producer.Return.Errors = true
saramaCfg.Producer.Compression = sarama.CompressionSnappy
saramaCfg.Producer.Retry.Backoff = 10 * time.Second
saramaCfg.Producer.Retry.Max = 6
saramaCfg.Producer.Flush.Frequency = 500 * time.Millisecond
saramaCfg.Producer.Flush.Bytes = 1024 * 1024
saramaClient, err := sarama.NewClient(cfg.Kafka.SeedPeers, saramaCfg)
if err != nil {
return nil, fmt.Errorf("failed to create sarama.Client, err=(%s)", err)
}
saramaProducer, err := sarama.NewAsyncProducerFromClient(saramaClient)
if err != nil {
return nil, fmt.Errorf("failed to create sarama.Producer, err=(%s)", err)
}
p := &T{
baseCID: sarama.RootCID.NewChild("producer"),
saramaClient: saramaClient,
saramaProducer: saramaProducer,
shutdownTimeout: cfg.Producer.ShutdownTimeout,
deadMessageCh: cfg.Producer.DeadMessageCh,
dispatcherCh: make(chan *sarama.ProducerMessage, cfg.Producer.ChannelBufferSize),
resultCh: make(chan produceResult, cfg.Producer.ChannelBufferSize),
}
spawn(&p.wg, p.merge)
spawn(&p.wg, p.dispatch)
return p, nil
}
func NewKafkaHelper(c *C) *KafkaHelper {
kh := &KafkaHelper{c: c}
cfg := sarama.NewConfig()
cfg.Producer.Return.Successes = true
cfg.Producer.Return.Errors = true
cfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
cfg.ClientID = "unittest-runner"
err := error(nil)
if kh.client, err = sarama.NewClient(KafkaPeers, cfg); err != nil {
panic(err)
}
if kh.consumer, err = sarama.NewConsumerFromClient(kh.client); err != nil {
panic(err)
}
if kh.producer, err = sarama.NewAsyncProducerFromClient(kh.client); err != nil {
panic(err)
}
if kh.offsetMgr, err = sarama.NewOffsetManagerFromClient(kh.client); err != nil {
panic(err)
}
return kh
}
// SpawnGracefulProducer creates a `KafkaProducer` instance and starts its internal
// goroutines.
func SpawnGracefulProducer(config *config.T) (*GracefulProducer, error) {
saramaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.SaramaConfig())
if err != nil {
return nil, fmt.Errorf("failed to create sarama.Client, err=(%s)", err)
}
saramaProducer, err := sarama.NewAsyncProducerFromClient(saramaClient)
if err != nil {
return nil, fmt.Errorf("failed to create sarama.Producer, err=(%s)", err)
}
gp := &GracefulProducer{
baseCID: sarama.RootCID.NewChild("producer"),
saramaClient: saramaClient,
saramaProducer: saramaProducer,
shutdownTimeout: config.Producer.ShutdownTimeout,
deadMessageCh: config.Producer.DeadMessageCh,
dispatcherCh: make(chan *sarama.ProducerMessage, config.ChannelBufferSize),
resultCh: make(chan produceResult, config.ChannelBufferSize),
}
spawn(&gp.wg, gp.merge)
spawn(&gp.wg, gp.dispatch)
return gp, nil
}