// Validate checks a Config instance. It will return a
// sarama.ConfigurationError if the specified values don't make sense.
func (c *Config) Validate() error {
if c.Group.Heartbeat.Interval%time.Millisecond != 0 {
sarama.Logger.Println("Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.")
}
if c.Group.Session.Timeout%time.Millisecond != 0 {
sarama.Logger.Println("Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.")
}
if c.Group.PartitionStrategy != StrategyRange && c.Group.PartitionStrategy != StrategyRoundRobin {
sarama.Logger.Println("Group.PartitionStrategy is not supported; range will be assumed.")
}
if err := c.Config.Validate(); err != nil {
return err
}
// validate the Group values
switch {
case c.Group.Offsets.Retry.Max < 0:
return sarama.ConfigurationError("Group.Offsets.Retry.Max must be >= 0")
case c.Group.Heartbeat.Interval <= 0:
return sarama.ConfigurationError("Group.Heartbeat.Interval must be > 0")
case c.Group.Session.Timeout <= 0:
return sarama.ConfigurationError("Group.Session.Timeout must be > 0")
}
// ensure offset is correct
switch c.Consumer.Offsets.Initial {
case sarama.OffsetOldest, sarama.OffsetNewest:
default:
return sarama.ConfigurationError("Consumer.Offsets.Initial must be either OffsetOldest or OffsetNewest")
}
return nil
}
// Validate consumer config, maybe sarama can expose a public ConsumerConfig.Validate() one day
func validateConsumerConfig(config *sarama.ConsumerConfig) error {
if config.DefaultFetchSize < 0 {
return sarama.ConfigurationError("Invalid DefaultFetchSize")
} else if config.DefaultFetchSize == 0 {
config.DefaultFetchSize = 1024
}
if config.MinFetchSize < 0 {
return sarama.ConfigurationError("Invalid MinFetchSize")
} else if config.MinFetchSize == 0 {
config.MinFetchSize = 1
}
if config.MaxWaitTime <= 0 {
return sarama.ConfigurationError("Invalid MaxWaitTime")
} else if config.MaxWaitTime < 100 {
sarama.Logger.Println("ConsumerConfig.MaxWaitTime is very low, which can cause high CPU and network usage. See sarama documentation for details.")
}
if config.MaxMessageSize < 0 {
return sarama.ConfigurationError("Invalid MaxMessageSize")
} else if config.EventBufferSize < 0 {
return sarama.ConfigurationError("Invalid EventBufferSize")
}
return nil
}
// NewConsumerGroup creates a new consumer group for a given topic.
//
// You MUST call Close() on a consumer to avoid leaks, it will not be garbage-collected automatically when
// it passes out of scope (this is in addition to calling Close on the underlying client, which is still necessary).
func NewConsumerGroup(client *sarama.Client, zoo *ZK, name string, topic string, logger Loggable, config *sarama.ConsumerConfig) (group *ConsumerGroup, err error) {
if config == nil {
config = new(sarama.ConsumerConfig)
}
// Validate configuration
if err = validateConsumerConfig(config); err != nil {
return
} else if topic == "" {
return nil, sarama.ConfigurationError("Empty topic")
} else if name == "" {
return nil, sarama.ConfigurationError("Empty name")
}
// Register consumer group
if err = zoo.RegisterGroup(name); err != nil {
return
}
// Init struct
group = &ConsumerGroup{
id: GUID.New(name),
name: name,
topic: topic,
config: config,
client: client,
zoo: zoo,
claims: make([]PartitionConsumer, 0),
logger: logger,
stopper: make(chan bool),
done: make(chan bool),
checkout: make(chan bool),
force: make(chan bool),
claimed: make(chan *PartitionConsumer),
}
// Register itself with zookeeper
if err = zoo.RegisterConsumer(group.name, group.id, group.topic); err != nil {
return nil, err
}
go group.signalLoop()
return group, nil
}
func (cgc *Config) Validate() error {
if cgc.Zookeeper.Timeout <= 0 {
return sarama.ConfigurationError("ZookeeperTimeout should have a duration > 0")
}
if cgc.Offsets.CommitInterval <= 0 {
return sarama.ConfigurationError("CommitInterval should have a duration > 0")
}
if cgc.Offsets.Initial != sarama.OffsetOldest && cgc.Offsets.Initial != sarama.OffsetNewest {
return errors.New("Offsets.Initial should be sarama.OffsetOldest or sarama.OffsetNewest.")
}
if cgc.Config != nil {
if err := cgc.Config.Validate(); err != nil {
return err
}
}
return nil
}
// NewProducer creates a new Producer using the given client, topic and configuration.
func NewProducer(client *sarama.Client, topic string, config *sarama.ProducerConfig) (*Producer, error) {
if topic == "" {
return nil, sarama.ConfigurationError("Empty topic")
}
prod, err := sarama.NewProducer(client, config)
if err != nil {
return nil, err
}
sp := &Producer{
producer: prod,
topic: topic,
newExpectations: make(chan *producerExpect), // this must be unbuffered
client: client,
}
return sp, nil
}
// ConsumePartition implements the ConsumePartition method from the sarama.Consumer interface.
// Before you can start consuming a partition, you have to set expectations on it using
// ExpectConsumePartition. You can only consume a partition once per consumer.
func (c *Consumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) {
c.l.Lock()
defer c.l.Unlock()
if c.partitionConsumers[topic] == nil || c.partitionConsumers[topic][partition] == nil {
c.t.Errorf("No expectations set for %s/%d", topic, partition)
return nil, errOutOfExpectations
}
pc := c.partitionConsumers[topic][partition]
if pc.consumed {
return nil, sarama.ConfigurationError("The topic/partition is already being consumed")
}
if pc.offset != AnyOffset && pc.offset != offset {
c.t.Errorf("Unexpected offset when calling ConsumePartition for %s/%d. Expected %d, got %d.", topic, partition, pc.offset, offset)
}
pc.consumed = true
return pc, nil
}
// Connects to a consumer group, using Zookeeper for auto-discovery
func JoinConsumerGroup(name string, topics []string, zookeeper []string, config *Config) (cg *ConsumerGroup, err error) {
if name == "" {
return nil, sarama.ConfigurationError("Empty consumergroup name")
}
if len(topics) == 0 {
return nil, sarama.ConfigurationError("No topics provided")
}
if len(zookeeper) == 0 {
return nil, errors.New("You need to provide at least one zookeeper node address!")
}
if config == nil {
config = NewConfig()
}
config.ClientID = name
// Validate configuration
if err = config.Validate(); err != nil {
return
}
var kz *kazoo.Kazoo
if kz, err = kazoo.NewKazoo(zookeeper, config.Zookeeper); err != nil {
return
}
brokers, err := kz.BrokerList()
if err != nil {
kz.Close()
return
}
group := kz.Consumergroup(name)
if config.Offsets.ResetOffsets {
err = group.ResetOffsets()
if err != nil {
cg.Logf("FAILED to reset offsets of consumergroup: %s!\n", err)
kz.Close()
return
}
}
instance := group.NewInstance()
var consumer sarama.Consumer
if consumer, err = sarama.NewConsumer(brokers, config.Config); err != nil {
kz.Close()
return
}
cg = &ConsumerGroup{
config: config,
consumer: consumer,
kazoo: kz,
group: group,
instance: instance,
messages: make(chan *sarama.ConsumerMessage, config.ChannelBufferSize),
errors: make(chan *sarama.ConsumerError, config.ChannelBufferSize),
stopper: make(chan struct{}),
}
// Register consumer group
if exists, err := cg.group.Exists(); err != nil {
cg.Logf("FAILED to check for existence of consumergroup: %s!\n", err)
_ = consumer.Close()
_ = kz.Close()
return nil, err
} else if !exists {
cg.Logf("Consumergroup `%s` does not yet exists, creating...\n", cg.group.Name)
if err := cg.group.Create(); err != nil {
cg.Logf("FAILED to create consumergroup in Zookeeper: %s!\n", err)
_ = consumer.Close()
_ = kz.Close()
return nil, err
}
}
// Register itself with zookeeper
if err := cg.instance.Register(topics); err != nil {
cg.Logf("FAILED to register consumer instance: %s!\n", err)
return nil, err
} else {
cg.Logf("Consumer instance registered (%s).", cg.instance.ID)
}
offsetConfig := OffsetManagerConfig{CommitInterval: config.Offsets.CommitInterval}
cg.offsetManager = NewZookeeperOffsetManager(cg, &offsetConfig)
go cg.topicListConsumer(topics)
return
}
func JoinConsumerGroupRealIp(realIp string, name string, topics []string, zookeeper []string,
config *Config) (cg *ConsumerGroup, err error) {
if name == "" {
return nil, sarama.ConfigurationError("Empty consumergroup name")
}
if len(topics) == 0 {
return nil, sarama.ConfigurationError("No topics provided")
}
if len(zookeeper) == 0 {
return nil, EmptyZkAddrs
}
if config == nil {
config = NewConfig()
}
config.ClientID = name
if err = config.Validate(); err != nil {
return
}
var kz *kazoo.Kazoo
if kz, err = kazoo.NewKazoo(zookeeper, config.Zookeeper); err != nil {
return
}
group := kz.Consumergroup(name)
if config.Offsets.ResetOffsets {
err = group.ResetOffsets()
if err != nil {
kz.Close()
return
}
}
instance := group.NewInstanceRealIp(realIp)
cg = &ConsumerGroup{
config: config,
kazoo: kz,
group: group,
instance: instance,
messages: make(chan *sarama.ConsumerMessage, config.ChannelBufferSize),
errors: make(chan *sarama.ConsumerError, config.ChannelBufferSize),
stopper: make(chan struct{}),
}
if config.NoDup {
cg.cacher = freecache.NewCache(1 << 20) // TODO
}
// Register consumer group in zookeeper
if exists, err := cg.group.Exists(); err != nil {
_ = kz.Close()
return nil, err
} else if !exists {
log.Debug("[%s/%s] consumer group in zk creating...", cg.group.Name, cg.shortID())
if err := cg.group.Create(); err != nil {
_ = kz.Close()
return nil, err
}
}
// Register itself with zookeeper: consumers/{group}/ids/{instanceId}
// This will lead to consumer group rebalance
if err := cg.instance.Register(topics); err != nil {
return nil, err
} else {
log.Debug("[%s/%s] cg instance registered in zk for %+v", cg.group.Name, cg.shortID(), topics)
}
// kafka connect
brokers, err := cg.kazoo.BrokerList()
if err != nil {
return nil, err
}
if consumer, err := sarama.NewConsumer(brokers, cg.config.Config); err != nil {
return nil, err
} else {
cg.consumer = consumer
}
offsetConfig := OffsetManagerConfig{CommitInterval: config.Offsets.CommitInterval}
cg.offsetManager = NewZookeeperOffsetManager(cg, &offsetConfig)
cg.wg.Add(1)
go cg.consumeTopics(topics)
return
}