Example #1
0
func (this *Mirror) makeSub(c1 *zk.ZkCluster, group string, topics []string) (*consumergroup.ConsumerGroup, error) {
	cf := consumergroup.NewConfig()
	cf.Zookeeper.Chroot = c1.Chroot()
	cf.Offsets.CommitInterval = time.Second * 10
	cf.Offsets.ProcessingTimeout = time.Second
	cf.ChannelBufferSize = 0
	cf.Consumer.Return.Errors = true
	cf.Consumer.MaxProcessingTime = 100 * time.Millisecond // chan recv timeout

	sub, err := consumergroup.JoinConsumerGroup(group, topics, c1.ZkZone().ZkAddrList(), cf)
	return sub, err
}
Example #2
0
func (this *Mirror) makeSub(c1 *zk.ZkCluster, group string, topics []string) (*consumergroup.ConsumerGroup, error) {
	cf := consumergroup.NewConfig()
	cf.Zookeeper.Chroot = c1.Chroot()
	cf.Offsets.CommitInterval = time.Second * 10
	cf.Offsets.ProcessingTimeout = time.Second
	cf.Consumer.Offsets.Initial = sarama.OffsetOldest
	cf.ChannelBufferSize = 256
	cf.Consumer.Return.Errors = true
	cf.OneToOne = false

	sub, err := consumergroup.JoinConsumerGroup(group, topics, c1.ZkZone().ZkAddrList(), cf)
	return sub, err
}
Example #3
0
func (this *WebhookExecutor) Run() {
	// TODO watch the znode change, its endpoint might change any time
	if len(this.endpoints) == 0 {
		log.Warn("%s disabled webhook: empty endpoints", this.topic)
		return
	}

	this.appid = manager.Default.TopicAppid(this.topic)
	if this.appid == "" {
		log.Warn("invalid topic: %s", this.topic)
		return
	}

	this.appSignature = manager.Default.Signature(this.appid)
	if this.appSignature == "" {
		log.Warn("%s/%s invalid app signature", this.topic, this.appid)
	}

	cf := consumergroup.NewConfig()
	cf.Net.DialTimeout = time.Second * 10
	cf.Net.WriteTimeout = time.Second * 10
	cf.Net.ReadTimeout = time.Second * 10
	cf.ChannelBufferSize = 100
	cf.Consumer.Return.Errors = true
	cf.Consumer.MaxProcessingTime = time.Second * 2 // chan recv timeout
	cf.Zookeeper.Chroot = meta.Default.ZkChroot(this.cluster)
	cf.Zookeeper.Timeout = zk.DefaultZkSessionTimeout()
	cf.Offsets.CommitInterval = time.Minute
	cf.Offsets.ProcessingTimeout = time.Second
	cf.Offsets.ResetOffsets = false
	cf.Offsets.Initial = sarama.OffsetOldest
	cg, err := consumergroup.JoinConsumerGroup(groupName, []string{this.topic}, meta.Default.ZkAddrs(), cf)
	if err != nil {
		log.Error("%s stopped: %s", this.topic, err)
		return
	}
	this.fetcher = cg

	var wg sync.WaitGroup
	for i := 0; i < 1; i++ {
		wg.Add(1)
		go this.pump(&wg)
	}

	for {
		select {
		case <-this.stopper:
			log.Debug("%s stopping", this.topic)
			wg.Wait()
			return

		case err := <-cg.Errors():
			log.Error("%s %s", this.topic, err)
			// TODO

		case msg := <-cg.Messages():
			this.msgCh <- msg
		}

	}

}