Beispiel #1
0
func (this *Mirror) makeSub(c1 *zk.ZkCluster, group string, topics []string) (*consumergroup.ConsumerGroup, error) {
	cf := consumergroup.NewConfig()
	cf.Zookeeper.Chroot = c1.Chroot()
	cf.Offsets.CommitInterval = time.Second * 10
	cf.Offsets.ProcessingTimeout = time.Second
	cf.ChannelBufferSize = 0
	cf.Consumer.Return.Errors = true
	cf.Consumer.MaxProcessingTime = 100 * time.Millisecond // chan recv timeout

	sub, err := consumergroup.JoinConsumerGroup(group, topics, c1.ZkZone().ZkAddrList(), cf)
	return sub, err
}
Beispiel #2
0
func (this *Mirror) makeSub(c1 *zk.ZkCluster, group string, topics []string) (*consumergroup.ConsumerGroup, error) {
	cf := consumergroup.NewConfig()
	cf.Zookeeper.Chroot = c1.Chroot()
	cf.Offsets.CommitInterval = time.Second * 10
	cf.Offsets.ProcessingTimeout = time.Second
	cf.Consumer.Offsets.Initial = sarama.OffsetOldest
	cf.ChannelBufferSize = 256
	cf.Consumer.Return.Errors = true
	cf.OneToOne = false

	sub, err := consumergroup.JoinConsumerGroup(group, topics, c1.ZkZone().ZkAddrList(), cf)
	return sub, err
}
Beispiel #3
0
func (this *WebhookExecutor) Run() {
	// TODO watch the znode change, its endpoint might change any time
	if len(this.endpoints) == 0 {
		log.Warn("%s disabled webhook: empty endpoints", this.topic)
		return
	}

	this.appid = manager.Default.TopicAppid(this.topic)
	if this.appid == "" {
		log.Warn("invalid topic: %s", this.topic)
		return
	}

	this.appSignature = manager.Default.Signature(this.appid)
	if this.appSignature == "" {
		log.Warn("%s/%s invalid app signature", this.topic, this.appid)
	}

	cf := consumergroup.NewConfig()
	cf.Net.DialTimeout = time.Second * 10
	cf.Net.WriteTimeout = time.Second * 10
	cf.Net.ReadTimeout = time.Second * 10
	cf.ChannelBufferSize = 100
	cf.Consumer.Return.Errors = true
	cf.Consumer.MaxProcessingTime = time.Second * 2 // chan recv timeout
	cf.Zookeeper.Chroot = meta.Default.ZkChroot(this.cluster)
	cf.Zookeeper.Timeout = zk.DefaultZkSessionTimeout()
	cf.Offsets.CommitInterval = time.Minute
	cf.Offsets.ProcessingTimeout = time.Second
	cf.Offsets.ResetOffsets = false
	cf.Offsets.Initial = sarama.OffsetOldest
	cg, err := consumergroup.JoinConsumerGroup(groupName, []string{this.topic}, meta.Default.ZkAddrs(), cf)
	if err != nil {
		log.Error("%s stopped: %s", this.topic, err)
		return
	}
	this.fetcher = cg

	var wg sync.WaitGroup
	for i := 0; i < 1; i++ {
		wg.Add(1)
		go this.pump(&wg)
	}

	for {
		select {
		case <-this.stopper:
			log.Debug("%s stopping", this.topic)
			wg.Wait()
			return

		case err := <-cg.Errors():
			log.Error("%s %s", this.topic, err)
			// TODO

		case msg := <-cg.Messages():
			this.msgCh <- msg
		}

	}

}
Beispiel #4
0
func (this *subManager) PickConsumerGroup(cluster, topic, group, remoteAddr, realIp string,
	resetOffset string, permitStandby bool) (cg *consumergroup.ConsumerGroup, err error) {
	// find consumger group from cache
	var present bool
	this.clientMapLock.RLock()
	cg, present = this.clientMap[remoteAddr]
	this.clientMapLock.RUnlock()
	if present {
		return
	}

	if !permitStandby {
		// ensure concurrent sub threads didn't exceed partition count
		// the 1st non-strict barrier, consumer group is the final barrier
		partitionN := len(meta.Default.TopicPartitions(cluster, topic))
		if partitionN == 0 {
			err = store.ErrInvalidTopic
			return
		}
		onlineN, e := meta.Default.OnlineConsumersCount(cluster, topic, group)
		if e != nil {
			return nil, e
		}
		if onlineN >= partitionN {
			err = store.ErrTooManyConsumers
			return
		}
	}

	this.clientMapLock.Lock()
	defer this.clientMapLock.Unlock()

	// double check lock
	cg, present = this.clientMap[remoteAddr]
	if present {
		return
	}

	// cache miss, create the consumer group for this client
	cf := consumergroup.NewConfig()
	cf.PermitStandby = permitStandby
	cf.OneToOne = true

	cf.Net.DialTimeout = time.Second * 10
	cf.Net.WriteTimeout = time.Second * 10
	cf.Net.ReadTimeout = time.Second * 10

	// kafka Fetch already batched into MessageSet,
	// this chan buf size influence on throughput is ignoreable
	cf.ChannelBufferSize = 0
	// kafka Fetch MaxWaitTime 250ms, MinByte=1 by default

	cf.Consumer.Return.Errors = true
	cf.Consumer.MaxProcessingTime = time.Second * 2 // chan recv timeout
	cf.Zookeeper.Chroot = meta.Default.ZkChroot(cluster)
	cf.Zookeeper.Timeout = zk.DefaultZkSessionTimeout()
	cf.Offsets.CommitInterval = time.Minute
	cf.Offsets.ProcessingTimeout = time.Second
	switch resetOffset {
	case "newest":
		cf.Offsets.ResetOffsets = true
		cf.Offsets.Initial = sarama.OffsetNewest
	case "oldest":
		cf.Offsets.ResetOffsets = true
		cf.Offsets.Initial = sarama.OffsetOldest
	default:
		cf.Offsets.ResetOffsets = false
		cf.Offsets.Initial = sarama.OffsetOldest
	}

	// runs in serial
	cg, err = consumergroup.JoinConsumerGroupRealIp(realIp, group, []string{topic},
		meta.Default.ZkAddrs(), cf)
	if err == nil {
		this.clientMap[remoteAddr] = cg
	}

	return
}