// Spawn creates a producer instance and starts its internal goroutines. func Spawn(cfg *config.T) (*T, error) { saramaCfg := sarama.NewConfig() saramaCfg.ChannelBufferSize = cfg.Producer.ChannelBufferSize saramaCfg.Producer.RequiredAcks = sarama.WaitForAll saramaCfg.Producer.Return.Successes = true saramaCfg.Producer.Return.Errors = true saramaCfg.Producer.Compression = sarama.CompressionSnappy saramaCfg.Producer.Retry.Backoff = 10 * time.Second saramaCfg.Producer.Retry.Max = 6 saramaCfg.Producer.Flush.Frequency = 500 * time.Millisecond saramaCfg.Producer.Flush.Bytes = 1024 * 1024 saramaClient, err := sarama.NewClient(cfg.Kafka.SeedPeers, saramaCfg) if err != nil { return nil, fmt.Errorf("failed to create sarama.Client, err=(%s)", err) } saramaProducer, err := sarama.NewAsyncProducerFromClient(saramaClient) if err != nil { return nil, fmt.Errorf("failed to create sarama.Producer, err=(%s)", err) } p := &T{ baseCID: sarama.RootCID.NewChild("producer"), saramaClient: saramaClient, saramaProducer: saramaProducer, shutdownTimeout: cfg.Producer.ShutdownTimeout, deadMessageCh: cfg.Producer.DeadMessageCh, dispatcherCh: make(chan *sarama.ProducerMessage, cfg.Producer.ChannelBufferSize), resultCh: make(chan produceResult, cfg.Producer.ChannelBufferSize), } spawn(&p.wg, p.merge) spawn(&p.wg, p.dispatch) return p, nil }
func NewTestKafkaClient(brokers []string) *TestKafkaClient { tkc := &TestKafkaClient{} clientCfg := sarama.NewConfig() clientCfg.ClientID = "unittest-runner" err := error(nil) if tkc.client, err = sarama.NewClient(brokers, clientCfg); err != nil { panic(err) } if tkc.consumer, err = sarama.NewConsumerFromClient(tkc.client); err != nil { panic(err) } return tkc }
// Messages that are larger then producer's MaxMessageBytes size are silently // dropped. Note that we assume that the broker's limit is the same as the // producer's one or higher. func (s *ServiceSuite) TestMessageTooLarge(c *C) { offsetsBefore := s.tkc.getOffsets("test.4") maxMsgSize := sarama.NewConfig().Producer.MaxMessageBytes - ProdMsgMetadataSize([]byte("foo")) + 1 msg := GenMessage(maxMsgSize) s.config.TCPAddr = "127.0.0.1:55501" svc, _ := SpawnService(s.config) // When r := PostChunked(s.tcpClient, "http://127.0.0.1:55501/topics/test.4/messages?key=foo", msg) svc.Stop() // Have to stop before getOffsets // Then c.Assert(r.StatusCode, Equals, http.StatusOK) c.Assert(ParseJSONBody(c, r), DeepEquals, map[string]interface{}{}) offsetsAfter := s.tkc.getOffsets("test.4") c.Assert(offsetsAfter, DeepEquals, offsetsBefore) }
// Spawn creates a consumer instance with the specified configuration and // starts all its goroutines. func Spawn(cfg *config.T) (*T, error) { saramaCfg := sarama.NewConfig() saramaCfg.ClientID = cfg.ClientID saramaCfg.ChannelBufferSize = cfg.Consumer.ChannelBufferSize saramaCfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond saramaCfg.Consumer.Retry.Backoff = cfg.Consumer.BackOffTimeout saramaCfg.Consumer.Fetch.Default = 1024 * 1024 kafkaClient, err := sarama.NewClient(cfg.Kafka.SeedPeers, saramaCfg) if err != nil { return nil, ErrSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err)) } offsetMgr, err := sarama.NewOffsetManagerFromClient(kafkaClient) if err != nil { return nil, ErrSetup(fmt.Errorf("failed to create sarama.OffsetManager: err=(%v)", err)) } kazooCfg := kazoo.NewConfig() kazooCfg.Chroot = cfg.ZooKeeper.Chroot // ZooKeeper documentation says following about the session timeout: "The // current (ZooKeeper) implementation requires that the timeout be a // minimum of 2 times the tickTime (as set in the server configuration) and // a maximum of 20 times the tickTime". The default tickTime is 2 seconds. // See http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#ch_zkSessions kazooCfg.Timeout = 15 * time.Second kazooConn, err := kazoo.NewKazoo(cfg.ZooKeeper.SeedPeers, kazooCfg) if err != nil { return nil, ErrSetup(fmt.Errorf("failed to create kazoo.Kazoo: err=(%v)", err)) } sc := &T{ baseCID: sarama.RootCID.NewChild("smartConsumer"), cfg: cfg, kafkaClient: kafkaClient, offsetMgr: offsetMgr, kazooConn: kazooConn, } sc.dispatcher = newDispatcher(sc.baseCID, sc, sc.cfg) sc.dispatcher.start() return sc, nil }
// saramaConfig generates a `Shopify/sarama` library config. func (c *Config) saramaConfig() *sarama.Config { saramaConfig := sarama.NewConfig() saramaConfig.ClientID = c.ClientID saramaConfig.ChannelBufferSize = c.ChannelBufferSize saramaConfig.Producer.RequiredAcks = sarama.WaitForAll saramaConfig.Producer.Return.Successes = true saramaConfig.Producer.Return.Errors = true saramaConfig.Producer.Compression = sarama.CompressionSnappy saramaConfig.Producer.Retry.Backoff = 4 * time.Second saramaConfig.Producer.Retry.Max = 5 saramaConfig.Producer.Flush.Frequency = 500 * time.Millisecond saramaConfig.Producer.Flush.Bytes = 1024 * 1024 saramaConfig.Consumer.Offsets.CommitInterval = 50 * time.Millisecond saramaConfig.Consumer.Retry.Backoff = c.Consumer.BackOffTimeout saramaConfig.Consumer.Fetch.Default = 1024 * 1024 return saramaConfig }
func NewKafkaHelper(c *C) *KafkaHelper { kh := &KafkaHelper{c: c} cfg := sarama.NewConfig() cfg.Producer.Return.Successes = true cfg.Producer.Return.Errors = true cfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond cfg.ClientID = "unittest-runner" err := error(nil) if kh.client, err = sarama.NewClient(KafkaPeers, cfg); err != nil { panic(err) } if kh.consumer, err = sarama.NewConsumerFromClient(kh.client); err != nil { panic(err) } if kh.producer, err = sarama.NewAsyncProducerFromClient(kh.client); err != nil { panic(err) } if kh.offsetMgr, err = sarama.NewOffsetManagerFromClient(kh.client); err != nil { panic(err) } return kh }
// saramaConfig generates a `Shopify/sarama` library config. func (a *T) saramaConfig() *sarama.Config { saramaConfig := sarama.NewConfig() saramaConfig.ClientID = a.cfg.ClientID return saramaConfig }