Example #1
0
func (this *Mirror) makeMirror(c1, c2 *zk.ZkCluster) {
	pub, err := this.makePub(c2)
	swallow(err)

	topics, topicsChanges, err := c1.WatchTopics()
	swallow(err)

	log.Printf("topics: %+v", topics)
	if len(topics) == 0 {
		log.Println("empty topics")
		return
	}

	group := fmt.Sprintf("%s.%s._mirror_", c1.Name(), c2.Name())
	sub, err := this.makeSub(c1, group, topics)
	swallow(err)

	pumpStopper := make(chan struct{})
	go this.pump(sub, pub, pumpStopper)

LOOP:
	for {
		select {
		case <-topicsChanges:
			log.Println("topics changed, stopping pump...")
			pumpStopper <- struct{}{} // stop pump
			<-pumpStopper             // await pump cleanup

			// refresh c1 topics
			topics, err = c1.Topics()
			if err != nil {
				// TODO how to handle this err?
				log.Println(err)
			}

			log.Printf("topics: %+v", topics)

			sub, err = this.makeSub(c1, group, topics)
			if err != nil {
				// TODO how to handle this err?
				log.Println(err)
			}

			go this.pump(sub, pub, pumpStopper)

		case <-this.quit:
			log.Println("awaiting pump cleanup...")
			<-pumpStopper
			log.Printf("total transferred: %s %smsgs",
				gofmt.ByteSize(this.transferBytes),
				gofmt.Comma(this.transferN))
			break LOOP
		}
	}

	pub.Close()
}
Example #2
0
func (this *Mirror) runMirror(c1, c2 *zk.ZkCluster, limit int64) {
	this.startedAt = time.Now()

	log.Info("start [%s/%s] -> [%s/%s] with bandwidth %sbps",
		c1.ZkZone().Name(), c1.Name(),
		c2.ZkZone().Name(), c2.Name(),
		gofmt.Comma(limit*8))

	pub, err := this.makePub(c2)
	if err != nil {
		panic(err)
	}
	log.Trace("pub[%s/%s] created", c2.ZkZone().Name(), c2.Name())

	go func(pub sarama.AsyncProducer, c *zk.ZkCluster) {
		for {
			select {
			case <-this.quit:
				return

			case err := <-pub.Errors():
				// messages will only be returned here after all retry attempts are exhausted.
				//
				// e,g
				// Failed to produce message to topic xx: write tcp src->kfk: i/o timeout
				// kafka: broker not connected
				log.Error("pub[%s/%s] %v", c.ZkZone().Name(), c.Name(), err)
			}
		}
	}(pub, c2)

	group := this.groupName(c1, c2)
	ever := true
	round := 0
	for ever {
		round++

		topics, topicsChanges, err := c1.WatchTopics()
		if err != nil {
			log.Error("#%d [%s/%s]watch topics: %v", round, c1.ZkZone().Name(), c1.Name(), err)
			time.Sleep(time.Second * 10)
			continue
		}

		topics = this.realTopics(topics)
		sub, err := this.makeSub(c1, group, topics)
		if err != nil {
			log.Error("#%d [%s/%s] %v", round, c1.ZkZone().Name(), c1.Name(), err)
			time.Sleep(time.Second * 10)
			continue
		}

		log.Info("#%d starting pump [%s/%s] -> [%s/%s] %d topics with group %s for %+v", round,
			c1.ZkZone().Name(), c1.Name(),
			c2.ZkZone().Name(), c2.Name(), len(topics), group, topics)

		pumpStopper := make(chan struct{})
		pumpStopped := make(chan struct{})
		go this.pump(sub, pub, pumpStopper, pumpStopped)

		select {
		case <-topicsChanges:
			// TODO log the diff the topics
			log.Warn("#%d [%s/%s] topics changed, stopping pump...", round, c1.Name(), c2.Name())
			pumpStopper <- struct{}{} // stop pump
			<-pumpStopped             // await pump cleanup

		case <-this.quit:
			log.Info("#%d awaiting pump cleanup...", round)
			<-pumpStopped

			ever = false

		case <-pumpStopped:
			// pump encounters problems, just retry
		}
	}

	log.Info("total transferred: %s %smsgs",
		gofmt.ByteSize(this.transferBytes),
		gofmt.Comma(this.transferN))

	log.Info("closing pub...")
	pub.Close()
}