// Spawn creates a producer instance and starts its internal goroutines.
func Spawn(cfg *config.T) (*T, error) {
saramaCfg := sarama.NewConfig()
saramaCfg.ChannelBufferSize = cfg.Producer.ChannelBufferSize
saramaCfg.Producer.RequiredAcks = sarama.WaitForAll
saramaCfg.Producer.Return.Successes = true
saramaCfg.Producer.Return.Errors = true
saramaCfg.Producer.Compression = sarama.CompressionSnappy
saramaCfg.Producer.Retry.Backoff = 10 * time.Second
saramaCfg.Producer.Retry.Max = 6
saramaCfg.Producer.Flush.Frequency = 500 * time.Millisecond
saramaCfg.Producer.Flush.Bytes = 1024 * 1024
saramaClient, err := sarama.NewClient(cfg.Kafka.SeedPeers, saramaCfg)
if err != nil {
return nil, fmt.Errorf("failed to create sarama.Client, err=(%s)", err)
}
saramaProducer, err := sarama.NewAsyncProducerFromClient(saramaClient)
if err != nil {
return nil, fmt.Errorf("failed to create sarama.Producer, err=(%s)", err)
}
p := &T{
baseCID: sarama.RootCID.NewChild("producer"),
saramaClient: saramaClient,
saramaProducer: saramaProducer,
shutdownTimeout: cfg.Producer.ShutdownTimeout,
deadMessageCh: cfg.Producer.DeadMessageCh,
dispatcherCh: make(chan *sarama.ProducerMessage, cfg.Producer.ChannelBufferSize),
resultCh: make(chan produceResult, cfg.Producer.ChannelBufferSize),
}
spawn(&p.wg, p.merge)
spawn(&p.wg, p.dispatch)
return p, nil
}
func NewTestKafkaClient(brokers []string) *TestKafkaClient {
tkc := &TestKafkaClient{}
clientCfg := sarama.NewConfig()
clientCfg.ClientID = "unittest-runner"
err := error(nil)
if tkc.client, err = sarama.NewClient(brokers, clientCfg); err != nil {
panic(err)
}
if tkc.consumer, err = sarama.NewConsumerFromClient(tkc.client); err != nil {
panic(err)
}
return tkc
}
// Messages that are larger then producer's MaxMessageBytes size are silently
// dropped. Note that we assume that the broker's limit is the same as the
// producer's one or higher.
func (s *ServiceSuite) TestMessageTooLarge(c *C) {
offsetsBefore := s.tkc.getOffsets("test.4")
maxMsgSize := sarama.NewConfig().Producer.MaxMessageBytes - ProdMsgMetadataSize([]byte("foo")) + 1
msg := GenMessage(maxMsgSize)
s.config.TCPAddr = "127.0.0.1:55501"
svc, _ := SpawnService(s.config)
// When
r := PostChunked(s.tcpClient, "http://127.0.0.1:55501/topics/test.4/messages?key=foo", msg)
svc.Stop() // Have to stop before getOffsets
// Then
c.Assert(r.StatusCode, Equals, http.StatusOK)
c.Assert(ParseJSONBody(c, r), DeepEquals, map[string]interface{}{})
offsetsAfter := s.tkc.getOffsets("test.4")
c.Assert(offsetsAfter, DeepEquals, offsetsBefore)
}
// Spawn creates a consumer instance with the specified configuration and
// starts all its goroutines.
func Spawn(cfg *config.T) (*T, error) {
saramaCfg := sarama.NewConfig()
saramaCfg.ClientID = cfg.ClientID
saramaCfg.ChannelBufferSize = cfg.Consumer.ChannelBufferSize
saramaCfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
saramaCfg.Consumer.Retry.Backoff = cfg.Consumer.BackOffTimeout
saramaCfg.Consumer.Fetch.Default = 1024 * 1024
kafkaClient, err := sarama.NewClient(cfg.Kafka.SeedPeers, saramaCfg)
if err != nil {
return nil, ErrSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err))
}
offsetMgr, err := sarama.NewOffsetManagerFromClient(kafkaClient)
if err != nil {
return nil, ErrSetup(fmt.Errorf("failed to create sarama.OffsetManager: err=(%v)", err))
}
kazooCfg := kazoo.NewConfig()
kazooCfg.Chroot = cfg.ZooKeeper.Chroot
// ZooKeeper documentation says following about the session timeout: "The
// current (ZooKeeper) implementation requires that the timeout be a
// minimum of 2 times the tickTime (as set in the server configuration) and
// a maximum of 20 times the tickTime". The default tickTime is 2 seconds.
// See http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#ch_zkSessions
kazooCfg.Timeout = 15 * time.Second
kazooConn, err := kazoo.NewKazoo(cfg.ZooKeeper.SeedPeers, kazooCfg)
if err != nil {
return nil, ErrSetup(fmt.Errorf("failed to create kazoo.Kazoo: err=(%v)", err))
}
sc := &T{
baseCID: sarama.RootCID.NewChild("smartConsumer"),
cfg: cfg,
kafkaClient: kafkaClient,
offsetMgr: offsetMgr,
kazooConn: kazooConn,
}
sc.dispatcher = newDispatcher(sc.baseCID, sc, sc.cfg)
sc.dispatcher.start()
return sc, nil
}
// saramaConfig generates a `Shopify/sarama` library config.
func (c *Config) saramaConfig() *sarama.Config {
saramaConfig := sarama.NewConfig()
saramaConfig.ClientID = c.ClientID
saramaConfig.ChannelBufferSize = c.ChannelBufferSize
saramaConfig.Producer.RequiredAcks = sarama.WaitForAll
saramaConfig.Producer.Return.Successes = true
saramaConfig.Producer.Return.Errors = true
saramaConfig.Producer.Compression = sarama.CompressionSnappy
saramaConfig.Producer.Retry.Backoff = 4 * time.Second
saramaConfig.Producer.Retry.Max = 5
saramaConfig.Producer.Flush.Frequency = 500 * time.Millisecond
saramaConfig.Producer.Flush.Bytes = 1024 * 1024
saramaConfig.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
saramaConfig.Consumer.Retry.Backoff = c.Consumer.BackOffTimeout
saramaConfig.Consumer.Fetch.Default = 1024 * 1024
return saramaConfig
}
func NewKafkaHelper(c *C) *KafkaHelper {
kh := &KafkaHelper{c: c}
cfg := sarama.NewConfig()
cfg.Producer.Return.Successes = true
cfg.Producer.Return.Errors = true
cfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
cfg.ClientID = "unittest-runner"
err := error(nil)
if kh.client, err = sarama.NewClient(KafkaPeers, cfg); err != nil {
panic(err)
}
if kh.consumer, err = sarama.NewConsumerFromClient(kh.client); err != nil {
panic(err)
}
if kh.producer, err = sarama.NewAsyncProducerFromClient(kh.client); err != nil {
panic(err)
}
if kh.offsetMgr, err = sarama.NewOffsetManagerFromClient(kh.client); err != nil {
panic(err)
}
return kh
}
// saramaConfig generates a `Shopify/sarama` library config.
func (a *T) saramaConfig() *sarama.Config {
saramaConfig := sarama.NewConfig()
saramaConfig.ClientID = a.cfg.ClientID
return saramaConfig
}