func (this *Mirror) pump(sub *consumergroup.ConsumerGroup, pub sarama.AsyncProducer, stop chan struct{}) { defer func() { log.Println("pump cleanup...") sub.Close() log.Println("pump cleanup ok") stop <- struct{}{} // notify others I'm done }() log.Printf("start pumping") active := false for { select { case <-this.quit: return case <-stop: // yes sir! return case <-time.After(time.Second * 10): active = false log.Println("idle 10s waiting for new msg") case msg := <-sub.Messages(): if !active || this.debug { log.Printf("<-[%d] T:%s M:%s", this.transferN, msg.Topic, string(msg.Value)) } active = true pub.Input() <- &sarama.ProducerMessage{ Topic: msg.Topic, Key: sarama.ByteEncoder(msg.Key), Value: sarama.ByteEncoder(msg.Value), } if this.autoCommit { sub.CommitUpto(msg) } // rate limit, never overflood the limited bandwidth between IDCs // FIXME when compressed, the bandwidth calculation is wrong bytesN := len(msg.Topic) + len(msg.Key) + len(msg.Value) + 20 // payload overhead if !this.bandwidthRateLimiter.Pour(bytesN) { time.Sleep(time.Second) this.Ui.Warn(fmt.Sprintf("%d -> bandwidth reached, backoff 1s", bytesN)) } this.transferBytes += int64(bytesN) this.transferN++ if this.transferN%this.progressStep == 0 { log.Println(gofmt.Comma(this.transferN)) } case err := <-sub.Errors(): this.Ui.Error(err.Error()) // TODO } } }
func (this *Mirror) pump(sub *consumergroup.ConsumerGroup, pub sarama.AsyncProducer, stop, stopped chan struct{}) { defer func() { log.Trace("closing sub, commit offsets...") sub.Close() stopped <- struct{}{} // notify others I'm done }() active := true backoff := time.Second * 2 idle := time.Second * 10 for { select { case <-this.quit: log.Trace("got signal quit") return case <-stop: // yes sir! log.Trace("got signal stop") return case <-time.After(idle): active = false log.Info("idle 10s waiting for new message") case msg, ok := <-sub.Messages(): if !ok { log.Warn("sub encounters end of message stream") return } if !active || this.Debug { log.Info("<-[#%d] T:%s M:%s", this.transferN, msg.Topic, string(msg.Value)) } active = true pub.Input() <- &sarama.ProducerMessage{ Topic: msg.Topic, Key: sarama.ByteEncoder(msg.Key), Value: sarama.ByteEncoder(msg.Value), } if this.AutoCommit { sub.CommitUpto(msg) } // rate limit, never overflood the limited bandwidth between IDCs // FIXME when compressed, the bandwidth calculation is wrong bytesN := len(msg.Topic) + len(msg.Key) + len(msg.Value) + 20 // payload overhead if this.bandwidthRateLimiter != nil && !this.bandwidthRateLimiter.Pour(bytesN) { log.Warn("%s -> bandwidth reached, backoff %s", gofmt.ByteSize(this.transferBytes), backoff) time.Sleep(backoff) } this.transferBytes += int64(bytesN) this.transferN++ if this.transferN%this.ProgressStep == 0 { log.Trace("%s %s %s", gofmt.Comma(this.transferN), gofmt.ByteSize(this.transferBytes), msg.Topic) } case err := <-sub.Errors(): log.Error("quitting pump %v", err) return } } }