// Spawn creates a consumer instance with the specified configuration and
// starts all its goroutines.
func Spawn(cfg *config.T) (*T, error) {
saramaCfg := sarama.NewConfig()
saramaCfg.ClientID = cfg.ClientID
saramaCfg.ChannelBufferSize = cfg.Consumer.ChannelBufferSize
saramaCfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
saramaCfg.Consumer.Retry.Backoff = cfg.Consumer.BackOffTimeout
saramaCfg.Consumer.Fetch.Default = 1024 * 1024
kafkaClient, err := sarama.NewClient(cfg.Kafka.SeedPeers, saramaCfg)
if err != nil {
return nil, ErrSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err))
}
offsetMgr, err := sarama.NewOffsetManagerFromClient(kafkaClient)
if err != nil {
return nil, ErrSetup(fmt.Errorf("failed to create sarama.OffsetManager: err=(%v)", err))
}
kazooCfg := kazoo.NewConfig()
kazooCfg.Chroot = cfg.ZooKeeper.Chroot
// ZooKeeper documentation says following about the session timeout: "The
// current (ZooKeeper) implementation requires that the timeout be a
// minimum of 2 times the tickTime (as set in the server configuration) and
// a maximum of 20 times the tickTime". The default tickTime is 2 seconds.
// See http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#ch_zkSessions
kazooCfg.Timeout = 15 * time.Second
kazooConn, err := kazoo.NewKazoo(cfg.ZooKeeper.SeedPeers, kazooCfg)
if err != nil {
return nil, ErrSetup(fmt.Errorf("failed to create kazoo.Kazoo: err=(%v)", err))
}
sc := &T{
baseCID: sarama.RootCID.NewChild("smartConsumer"),
cfg: cfg,
kafkaClient: kafkaClient,
offsetMgr: offsetMgr,
kazooConn: kazooConn,
}
sc.dispatcher = newDispatcher(sc.baseCID, sc, sc.cfg)
sc.dispatcher.start()
return sc, nil
}
func NewKafkaHelper(c *C) *KafkaHelper {
kh := &KafkaHelper{c: c}
cfg := sarama.NewConfig()
cfg.Producer.Return.Successes = true
cfg.Producer.Return.Errors = true
cfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond
cfg.ClientID = "unittest-runner"
err := error(nil)
if kh.client, err = sarama.NewClient(KafkaPeers, cfg); err != nil {
panic(err)
}
if kh.consumer, err = sarama.NewConsumerFromClient(kh.client); err != nil {
panic(err)
}
if kh.producer, err = sarama.NewAsyncProducerFromClient(kh.client); err != nil {
panic(err)
}
if kh.offsetMgr, err = sarama.NewOffsetManagerFromClient(kh.client); err != nil {
panic(err)
}
return kh
}
func ResetOffsets(c *C, group, topic string) {
config := NewConfig()
config.Kafka.SeedPeers = testKafkaPeers
config.ZooKeeper.SeedPeers = testZookeeperPeers
kafkaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.saramaConfig())
c.Assert(err, IsNil)
defer kafkaClient.Close()
offsetManager, err := sarama.NewOffsetManagerFromClient(kafkaClient)
c.Assert(err, IsNil)
partitions, err := kafkaClient.Partitions(topic)
c.Assert(err, IsNil)
for _, p := range partitions {
offset, err := kafkaClient.GetOffset(topic, p, sarama.OffsetNewest)
c.Assert(err, IsNil)
pom, err := offsetManager.ManagePartition(group, topic, p)
c.Assert(err, IsNil)
pom.SubmitOffset(offset, "dummy")
log.Infof("Set initial offset %s/%s/%d=%d", group, topic, p, offset)
pom.Close()
}
offsetManager.Close()
}
// SpawnSmartConsumer creates a SmartConsumer instance with the specified
// configuration and starts all its goroutines.
func SpawnSmartConsumer(config *Config) (*SmartConsumer, error) {
kafkaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.saramaConfig())
if err != nil {
return nil, ErrConsumerSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err))
}
offsetMgr, err := sarama.NewOffsetManagerFromClient(kafkaClient)
if err != nil {
return nil, ErrConsumerSetup(fmt.Errorf("failed to create sarama.OffsetManager: err=(%v)", err))
}
kazooConn, err := kazoo.NewKazoo(config.ZooKeeper.SeedPeers, config.kazooConfig())
if err != nil {
return nil, ErrConsumerSetup(fmt.Errorf("failed to create kazoo.Kazoo: err=(%v)", err))
}
sc := &SmartConsumer{
baseCID: sarama.RootCID.NewChild("smartConsumer"),
config: config,
kafkaClient: kafkaClient,
offsetMgr: offsetMgr,
kazooConn: kazooConn,
}
sc.dispatcher = newDispatcher(sc.baseCID, sc, sc.config)
sc.dispatcher.start()
return sc, nil
}