func (this *Mirror) makeMirror(c1, c2 *zk.ZkCluster) {
pub, err := this.makePub(c2)
swallow(err)
topics, topicsChanges, err := c1.WatchTopics()
swallow(err)
log.Printf("topics: %+v", topics)
if len(topics) == 0 {
log.Println("empty topics")
return
}
group := fmt.Sprintf("%s.%s._mirror_", c1.Name(), c2.Name())
sub, err := this.makeSub(c1, group, topics)
swallow(err)
pumpStopper := make(chan struct{})
go this.pump(sub, pub, pumpStopper)
LOOP:
for {
select {
case <-topicsChanges:
log.Println("topics changed, stopping pump...")
pumpStopper <- struct{}{} // stop pump
<-pumpStopper // await pump cleanup
// refresh c1 topics
topics, err = c1.Topics()
if err != nil {
// TODO how to handle this err?
log.Println(err)
}
log.Printf("topics: %+v", topics)
sub, err = this.makeSub(c1, group, topics)
if err != nil {
// TODO how to handle this err?
log.Println(err)
}
go this.pump(sub, pub, pumpStopper)
case <-this.quit:
log.Println("awaiting pump cleanup...")
<-pumpStopper
log.Printf("total transferred: %s %smsgs",
gofmt.ByteSize(this.transferBytes),
gofmt.Comma(this.transferN))
break LOOP
}
}
pub.Close()
}
func (this *Mirror) runMirror(c1, c2 *zk.ZkCluster, limit int64) {
this.startedAt = time.Now()
log.Info("start [%s/%s] -> [%s/%s] with bandwidth %sbps",
c1.ZkZone().Name(), c1.Name(),
c2.ZkZone().Name(), c2.Name(),
gofmt.Comma(limit*8))
pub, err := this.makePub(c2)
if err != nil {
panic(err)
}
log.Trace("pub[%s/%s] created", c2.ZkZone().Name(), c2.Name())
go func(pub sarama.AsyncProducer, c *zk.ZkCluster) {
for {
select {
case <-this.quit:
return
case err := <-pub.Errors():
// messages will only be returned here after all retry attempts are exhausted.
//
// e,g
// Failed to produce message to topic xx: write tcp src->kfk: i/o timeout
// kafka: broker not connected
log.Error("pub[%s/%s] %v", c.ZkZone().Name(), c.Name(), err)
}
}
}(pub, c2)
group := this.groupName(c1, c2)
ever := true
round := 0
for ever {
round++
topics, topicsChanges, err := c1.WatchTopics()
if err != nil {
log.Error("#%d [%s/%s]watch topics: %v", round, c1.ZkZone().Name(), c1.Name(), err)
time.Sleep(time.Second * 10)
continue
}
topics = this.realTopics(topics)
sub, err := this.makeSub(c1, group, topics)
if err != nil {
log.Error("#%d [%s/%s] %v", round, c1.ZkZone().Name(), c1.Name(), err)
time.Sleep(time.Second * 10)
continue
}
log.Info("#%d starting pump [%s/%s] -> [%s/%s] %d topics with group %s for %+v", round,
c1.ZkZone().Name(), c1.Name(),
c2.ZkZone().Name(), c2.Name(), len(topics), group, topics)
pumpStopper := make(chan struct{})
pumpStopped := make(chan struct{})
go this.pump(sub, pub, pumpStopper, pumpStopped)
select {
case <-topicsChanges:
// TODO log the diff the topics
log.Warn("#%d [%s/%s] topics changed, stopping pump...", round, c1.Name(), c2.Name())
pumpStopper <- struct{}{} // stop pump
<-pumpStopped // await pump cleanup
case <-this.quit:
log.Info("#%d awaiting pump cleanup...", round)
<-pumpStopped
ever = false
case <-pumpStopped:
// pump encounters problems, just retry
}
}
log.Info("total transferred: %s %smsgs",
gofmt.ByteSize(this.transferBytes),
gofmt.Comma(this.transferN))
log.Info("closing pub...")
pub.Close()
}