// Validate checks a Config instance. It will return a // sarama.ConfigurationError if the specified values don't make sense. func (c *Config) Validate() error { if c.Group.Heartbeat.Interval%time.Millisecond != 0 { sarama.Logger.Println("Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.") } if c.Group.Session.Timeout%time.Millisecond != 0 { sarama.Logger.Println("Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.") } if c.Group.PartitionStrategy != StrategyRange && c.Group.PartitionStrategy != StrategyRoundRobin { sarama.Logger.Println("Group.PartitionStrategy is not supported; range will be assumed.") } if err := c.Config.Validate(); err != nil { return err } // validate the Group values switch { case c.Group.Offsets.Retry.Max < 0: return sarama.ConfigurationError("Group.Offsets.Retry.Max must be >= 0") case c.Group.Heartbeat.Interval <= 0: return sarama.ConfigurationError("Group.Heartbeat.Interval must be > 0") case c.Group.Session.Timeout <= 0: return sarama.ConfigurationError("Group.Session.Timeout must be > 0") } // ensure offset is correct switch c.Consumer.Offsets.Initial { case sarama.OffsetOldest, sarama.OffsetNewest: default: return sarama.ConfigurationError("Consumer.Offsets.Initial must be either OffsetOldest or OffsetNewest") } return nil }
// Validate consumer config, maybe sarama can expose a public ConsumerConfig.Validate() one day func validateConsumerConfig(config *sarama.ConsumerConfig) error { if config.DefaultFetchSize < 0 { return sarama.ConfigurationError("Invalid DefaultFetchSize") } else if config.DefaultFetchSize == 0 { config.DefaultFetchSize = 1024 } if config.MinFetchSize < 0 { return sarama.ConfigurationError("Invalid MinFetchSize") } else if config.MinFetchSize == 0 { config.MinFetchSize = 1 } if config.MaxWaitTime <= 0 { return sarama.ConfigurationError("Invalid MaxWaitTime") } else if config.MaxWaitTime < 100 { sarama.Logger.Println("ConsumerConfig.MaxWaitTime is very low, which can cause high CPU and network usage. See sarama documentation for details.") } if config.MaxMessageSize < 0 { return sarama.ConfigurationError("Invalid MaxMessageSize") } else if config.EventBufferSize < 0 { return sarama.ConfigurationError("Invalid EventBufferSize") } return nil }
// NewConsumerGroup creates a new consumer group for a given topic. // // You MUST call Close() on a consumer to avoid leaks, it will not be garbage-collected automatically when // it passes out of scope (this is in addition to calling Close on the underlying client, which is still necessary). func NewConsumerGroup(client *sarama.Client, zoo *ZK, name string, topic string, logger Loggable, config *sarama.ConsumerConfig) (group *ConsumerGroup, err error) { if config == nil { config = new(sarama.ConsumerConfig) } // Validate configuration if err = validateConsumerConfig(config); err != nil { return } else if topic == "" { return nil, sarama.ConfigurationError("Empty topic") } else if name == "" { return nil, sarama.ConfigurationError("Empty name") } // Register consumer group if err = zoo.RegisterGroup(name); err != nil { return } // Init struct group = &ConsumerGroup{ id: GUID.New(name), name: name, topic: topic, config: config, client: client, zoo: zoo, claims: make([]PartitionConsumer, 0), logger: logger, stopper: make(chan bool), done: make(chan bool), checkout: make(chan bool), force: make(chan bool), claimed: make(chan *PartitionConsumer), } // Register itself with zookeeper if err = zoo.RegisterConsumer(group.name, group.id, group.topic); err != nil { return nil, err } go group.signalLoop() return group, nil }
func (cgc *Config) Validate() error { if cgc.Zookeeper.Timeout <= 0 { return sarama.ConfigurationError("ZookeeperTimeout should have a duration > 0") } if cgc.Offsets.CommitInterval <= 0 { return sarama.ConfigurationError("CommitInterval should have a duration > 0") } if cgc.Offsets.Initial != sarama.OffsetOldest && cgc.Offsets.Initial != sarama.OffsetNewest { return errors.New("Offsets.Initial should be sarama.OffsetOldest or sarama.OffsetNewest.") } if cgc.Config != nil { if err := cgc.Config.Validate(); err != nil { return err } } return nil }
// NewProducer creates a new Producer using the given client, topic and configuration. func NewProducer(client *sarama.Client, topic string, config *sarama.ProducerConfig) (*Producer, error) { if topic == "" { return nil, sarama.ConfigurationError("Empty topic") } prod, err := sarama.NewProducer(client, config) if err != nil { return nil, err } sp := &Producer{ producer: prod, topic: topic, newExpectations: make(chan *producerExpect), // this must be unbuffered client: client, } return sp, nil }
// ConsumePartition implements the ConsumePartition method from the sarama.Consumer interface. // Before you can start consuming a partition, you have to set expectations on it using // ExpectConsumePartition. You can only consume a partition once per consumer. func (c *Consumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) { c.l.Lock() defer c.l.Unlock() if c.partitionConsumers[topic] == nil || c.partitionConsumers[topic][partition] == nil { c.t.Errorf("No expectations set for %s/%d", topic, partition) return nil, errOutOfExpectations } pc := c.partitionConsumers[topic][partition] if pc.consumed { return nil, sarama.ConfigurationError("The topic/partition is already being consumed") } if pc.offset != AnyOffset && pc.offset != offset { c.t.Errorf("Unexpected offset when calling ConsumePartition for %s/%d. Expected %d, got %d.", topic, partition, pc.offset, offset) } pc.consumed = true return pc, nil }
// Connects to a consumer group, using Zookeeper for auto-discovery func JoinConsumerGroup(name string, topics []string, zookeeper []string, config *Config) (cg *ConsumerGroup, err error) { if name == "" { return nil, sarama.ConfigurationError("Empty consumergroup name") } if len(topics) == 0 { return nil, sarama.ConfigurationError("No topics provided") } if len(zookeeper) == 0 { return nil, errors.New("You need to provide at least one zookeeper node address!") } if config == nil { config = NewConfig() } config.ClientID = name // Validate configuration if err = config.Validate(); err != nil { return } var kz *kazoo.Kazoo if kz, err = kazoo.NewKazoo(zookeeper, config.Zookeeper); err != nil { return } brokers, err := kz.BrokerList() if err != nil { kz.Close() return } group := kz.Consumergroup(name) if config.Offsets.ResetOffsets { err = group.ResetOffsets() if err != nil { cg.Logf("FAILED to reset offsets of consumergroup: %s!\n", err) kz.Close() return } } instance := group.NewInstance() var consumer sarama.Consumer if consumer, err = sarama.NewConsumer(brokers, config.Config); err != nil { kz.Close() return } cg = &ConsumerGroup{ config: config, consumer: consumer, kazoo: kz, group: group, instance: instance, messages: make(chan *sarama.ConsumerMessage, config.ChannelBufferSize), errors: make(chan *sarama.ConsumerError, config.ChannelBufferSize), stopper: make(chan struct{}), } // Register consumer group if exists, err := cg.group.Exists(); err != nil { cg.Logf("FAILED to check for existence of consumergroup: %s!\n", err) _ = consumer.Close() _ = kz.Close() return nil, err } else if !exists { cg.Logf("Consumergroup `%s` does not yet exists, creating...\n", cg.group.Name) if err := cg.group.Create(); err != nil { cg.Logf("FAILED to create consumergroup in Zookeeper: %s!\n", err) _ = consumer.Close() _ = kz.Close() return nil, err } } // Register itself with zookeeper if err := cg.instance.Register(topics); err != nil { cg.Logf("FAILED to register consumer instance: %s!\n", err) return nil, err } else { cg.Logf("Consumer instance registered (%s).", cg.instance.ID) } offsetConfig := OffsetManagerConfig{CommitInterval: config.Offsets.CommitInterval} cg.offsetManager = NewZookeeperOffsetManager(cg, &offsetConfig) go cg.topicListConsumer(topics) return }
func JoinConsumerGroupRealIp(realIp string, name string, topics []string, zookeeper []string, config *Config) (cg *ConsumerGroup, err error) { if name == "" { return nil, sarama.ConfigurationError("Empty consumergroup name") } if len(topics) == 0 { return nil, sarama.ConfigurationError("No topics provided") } if len(zookeeper) == 0 { return nil, EmptyZkAddrs } if config == nil { config = NewConfig() } config.ClientID = name if err = config.Validate(); err != nil { return } var kz *kazoo.Kazoo if kz, err = kazoo.NewKazoo(zookeeper, config.Zookeeper); err != nil { return } group := kz.Consumergroup(name) if config.Offsets.ResetOffsets { err = group.ResetOffsets() if err != nil { kz.Close() return } } instance := group.NewInstanceRealIp(realIp) cg = &ConsumerGroup{ config: config, kazoo: kz, group: group, instance: instance, messages: make(chan *sarama.ConsumerMessage, config.ChannelBufferSize), errors: make(chan *sarama.ConsumerError, config.ChannelBufferSize), stopper: make(chan struct{}), } if config.NoDup { cg.cacher = freecache.NewCache(1 << 20) // TODO } // Register consumer group in zookeeper if exists, err := cg.group.Exists(); err != nil { _ = kz.Close() return nil, err } else if !exists { log.Debug("[%s/%s] consumer group in zk creating...", cg.group.Name, cg.shortID()) if err := cg.group.Create(); err != nil { _ = kz.Close() return nil, err } } // Register itself with zookeeper: consumers/{group}/ids/{instanceId} // This will lead to consumer group rebalance if err := cg.instance.Register(topics); err != nil { return nil, err } else { log.Debug("[%s/%s] cg instance registered in zk for %+v", cg.group.Name, cg.shortID(), topics) } // kafka connect brokers, err := cg.kazoo.BrokerList() if err != nil { return nil, err } if consumer, err := sarama.NewConsumer(brokers, cg.config.Config); err != nil { return nil, err } else { cg.consumer = consumer } offsetConfig := OffsetManagerConfig{CommitInterval: config.Offsets.CommitInterval} cg.offsetManager = NewZookeeperOffsetManager(cg, &offsetConfig) cg.wg.Add(1) go cg.consumeTopics(topics) return }