// Spawn creates a consumer instance with the specified configuration and // starts all its goroutines. func Spawn(cfg *config.T) (*T, error) { saramaCfg := sarama.NewConfig() saramaCfg.ClientID = cfg.ClientID saramaCfg.ChannelBufferSize = cfg.Consumer.ChannelBufferSize saramaCfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond saramaCfg.Consumer.Retry.Backoff = cfg.Consumer.BackOffTimeout saramaCfg.Consumer.Fetch.Default = 1024 * 1024 kafkaClient, err := sarama.NewClient(cfg.Kafka.SeedPeers, saramaCfg) if err != nil { return nil, ErrSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err)) } offsetMgr, err := sarama.NewOffsetManagerFromClient(kafkaClient) if err != nil { return nil, ErrSetup(fmt.Errorf("failed to create sarama.OffsetManager: err=(%v)", err)) } kazooCfg := kazoo.NewConfig() kazooCfg.Chroot = cfg.ZooKeeper.Chroot // ZooKeeper documentation says following about the session timeout: "The // current (ZooKeeper) implementation requires that the timeout be a // minimum of 2 times the tickTime (as set in the server configuration) and // a maximum of 20 times the tickTime". The default tickTime is 2 seconds. // See http://zookeeper.apache.org/doc/trunk/zookeeperProgrammers.html#ch_zkSessions kazooCfg.Timeout = 15 * time.Second kazooConn, err := kazoo.NewKazoo(cfg.ZooKeeper.SeedPeers, kazooCfg) if err != nil { return nil, ErrSetup(fmt.Errorf("failed to create kazoo.Kazoo: err=(%v)", err)) } sc := &T{ baseCID: sarama.RootCID.NewChild("smartConsumer"), cfg: cfg, kafkaClient: kafkaClient, offsetMgr: offsetMgr, kazooConn: kazooConn, } sc.dispatcher = newDispatcher(sc.baseCID, sc, sc.cfg) sc.dispatcher.start() return sc, nil }
func NewKafkaHelper(c *C) *KafkaHelper { kh := &KafkaHelper{c: c} cfg := sarama.NewConfig() cfg.Producer.Return.Successes = true cfg.Producer.Return.Errors = true cfg.Consumer.Offsets.CommitInterval = 50 * time.Millisecond cfg.ClientID = "unittest-runner" err := error(nil) if kh.client, err = sarama.NewClient(KafkaPeers, cfg); err != nil { panic(err) } if kh.consumer, err = sarama.NewConsumerFromClient(kh.client); err != nil { panic(err) } if kh.producer, err = sarama.NewAsyncProducerFromClient(kh.client); err != nil { panic(err) } if kh.offsetMgr, err = sarama.NewOffsetManagerFromClient(kh.client); err != nil { panic(err) } return kh }
func ResetOffsets(c *C, group, topic string) { config := NewConfig() config.Kafka.SeedPeers = testKafkaPeers config.ZooKeeper.SeedPeers = testZookeeperPeers kafkaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.saramaConfig()) c.Assert(err, IsNil) defer kafkaClient.Close() offsetManager, err := sarama.NewOffsetManagerFromClient(kafkaClient) c.Assert(err, IsNil) partitions, err := kafkaClient.Partitions(topic) c.Assert(err, IsNil) for _, p := range partitions { offset, err := kafkaClient.GetOffset(topic, p, sarama.OffsetNewest) c.Assert(err, IsNil) pom, err := offsetManager.ManagePartition(group, topic, p) c.Assert(err, IsNil) pom.SubmitOffset(offset, "dummy") log.Infof("Set initial offset %s/%s/%d=%d", group, topic, p, offset) pom.Close() } offsetManager.Close() }
// SpawnSmartConsumer creates a SmartConsumer instance with the specified // configuration and starts all its goroutines. func SpawnSmartConsumer(config *Config) (*SmartConsumer, error) { kafkaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.saramaConfig()) if err != nil { return nil, ErrConsumerSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err)) } offsetMgr, err := sarama.NewOffsetManagerFromClient(kafkaClient) if err != nil { return nil, ErrConsumerSetup(fmt.Errorf("failed to create sarama.OffsetManager: err=(%v)", err)) } kazooConn, err := kazoo.NewKazoo(config.ZooKeeper.SeedPeers, config.kazooConfig()) if err != nil { return nil, ErrConsumerSetup(fmt.Errorf("failed to create kazoo.Kazoo: err=(%v)", err)) } sc := &SmartConsumer{ baseCID: sarama.RootCID.NewChild("smartConsumer"), config: config, kafkaClient: kafkaClient, offsetMgr: offsetMgr, kazooConn: kazooConn, } sc.dispatcher = newDispatcher(sc.baseCID, sc, sc.config) sc.dispatcher.start() return sc, nil }