func (z *Zookeeper) updateCgAndWatch(cg string) {
path := fmt.Sprintf("%s/consumers/%s/offsets", z.cc.chroot, cg)
for {
children, _, eventChan, err := z.zkConn.ChildrenW(path)
if err == zk.ErrNoNode {
_, _, eventChan, err = z.zkConn.ExistsW(path)
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.ExistsW: path=%s, %s", path, err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
<-eventChan
continue
}
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.ChildrenW: path=%s, %s", path, err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
if len(children) == 0 {
logger.Warning("no transfer topic found: path=%s", path)
} else {
for _, topic := range children {
go z.updateTopicAndWatch(cg, topic)
}
}
event := <-eventChan
// let root watcher to re-init
if event.Type == zk.EventNotWatching {
return
}
}
}
/* do not change state in this func */
func (td *TransDi) processAckData(ackData *AckData) {
transid := ackData.transid
workerId := ackData.workerId
logger.Debug("process ack: transid=%d, workerId=%d", transid, workerId)
/* narrow window */
msg := td.transWindow.ackOne(transid)
if msg != nil && msg.Offset > td.ackedMinTransid {
td.ackedMinTransid = msg.Offset
err := td.cg.CommitUpto(msg)
if err != nil {
// TODO: optimized
logger.Warning("fail to consumergroup.CommitUpto(): %s", err.Error())
td.fatalErrorChan <- err
return
}
logger.Debug("consumergroup.CommitUpTo %d", msg.Offset)
}
logger.Debug("transWindow size: %d", td.transWindow.window.Len())
/* move worker */
worker := td.transWorkers[workerId]
worker.inWork = false
err := worker.workIfNeed()
if err != nil {
logger.Fatal("fail to let worker to work: %s", err.Error())
td.fatalErrorChan <- err
return
}
}
func (z *Zookeeper) updateBrokerListAndWatch() {
for {
children, _, eventChan, err := z.zkConn.ChildrenW(fmt.Sprintf("%s/brokers/ids", z.cc.chroot))
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.ChildrenW: %s", err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
if len(children) == 0 {
logger.Warning("no broker found")
} else {
logger.Debug("using child: %s", children[0])
val, _, err := z.zkConn.Get(fmt.Sprintf("%s/brokers/ids/%s", z.cc.chroot, children[0]))
var brokerIdVal BrokerIdVal
err = json.Unmarshal(val, &brokerIdVal)
if err != nil {
logger.Warning("fail to json.Unmarshal for broker id %s: %s", children[0], err.Error())
}
logger.Debug("broker id %s: host [%s] port [%d]", children[0], brokerIdVal.Host, brokerIdVal.Port)
z.brokerIdValChan <- &brokerIdVal
}
<-eventChan
}
}
func (z *Zookeeper) updatePartitionAndWatch(cg, topic string, partition int32) {
path := fmt.Sprintf("%s/consumers/%s/offsets/%s/%d", z.cc.chroot, cg, topic, partition)
for {
val, _, eventChan, err := z.zkConn.GetW(path)
if err == zk.ErrNoNode {
_, _, eventChan, err = z.zkConn.ExistsW(path)
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.ExistsW: path=%s, %s", path, err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
<-eventChan
continue
}
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.GetW: path=%s, %s", path, err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
if len(val) == 0 {
logger.Warning("transfer partition val is empty: path=%s", path)
} else {
offset, err := strconv.ParseInt(string(val), 10, 64)
if err != nil {
err = errors.New(fmt.Sprintf("fail to transfer: strconv.ParseInt: %s", err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
z.transferOffsetChan <- &TransferOffset{
cg: cg,
topic: topic,
partition: partition,
offset: offset,
}
logger.Debug("see transfer: cg=%s, topic=%s, partition=%d, offset=%d", cg, topic, partition, offset)
}
event := <-eventChan
// let root watcher to re-init
if event.Type == zk.EventNotWatching {
return
}
}
}
func (kp *KProxy) Run() {
go kp.producer.run()
go kp.httpServerRun()
err := <-kp.fatalErrorChan
logger.Fatal("%s", (*err).Error())
time.Sleep(1)
os.Exit(1)
}
func (bs *BinStore) Run() {
go bs.broker.run()
go bs.zk.run()
go bs.runHttpServer()
err := <-bs.fatalErrorChan
logger.Fatal("Fail: %s", err.Error())
return
}
func (km *KMonitor) Run() {
for _, c := range km.clusters {
go c.run()
}
go km.processFatalError()
err := km.server.ListenAndServe()
if err != nil {
logger.Fatal(err.Error())
}
}
func (httpServer *HttpServer) run() {
err := httpServer.server.ListenAndServe()
if err != nil {
logger.Fatal("fail to start http server: %s", err.Error())
httpServer.fatalErrorChan <- &err
return
}
err = errors.New("http server done")
httpServer.fatalErrorChan <- &err
}
func (kp *KProxy) httpServerRun() {
err := kp.server.ListenAndServe()
if err != nil {
logger.Fatal("fail to start http server: %s", err.Error())
kp.fatalErrorChan <- &err
return
}
err = errors.New("http server done")
kp.fatalErrorChan <- &err
}
func (bs *BinStore) runHttpServer() {
err := bs.server.ListenAndServe()
if err != nil {
logger.Fatal("fail to start http server: %s", err.Error())
bs.fatalErrorChan <- err
return
}
err = errors.New("http server done")
bs.fatalErrorChan <- err
}
func (transfer *KTransfer) Run() {
logger.Notice("ktransfer start")
for _, module := range transfer.modules {
go module.run()
}
for {
select {
case err := <-transfer.fatalErrorChan:
logger.Fatal("%s", err.Error())
return
}
}
return
}
func (b *Broker) updateBrokerClient(brokerIdVal *BrokerIdVal) {
if b.client != nil && b.client.Closed() == false {
b.client.Close()
}
b.client = nil
var err error
b.client, err = sarama.NewClient([]string{fmt.Sprintf("%s:%d", brokerIdVal.Host, brokerIdVal.Port)}, b.brokerConfig)
if err != nil {
err = errors.New(fmt.Sprintf("fail to sarama.NewClient: %s", err.Error()))
logger.Fatal(err.Error())
b.fatalErrorChan <- err
return
}
logger.Debug("success update broker client: %s:%d", brokerIdVal.Host, brokerIdVal.Port)
}
func (z *Zookeeper) updateTransferAndWatch() {
path := fmt.Sprintf("%s/consumers", z.cc.chroot)
for {
children, _, eventChan, err := z.zkConn.ChildrenW(path)
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.ChildrenW: path=%s, %s", path, err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
if len(children) == 0 {
logger.Warning("no transfer cg found: path=%s", path)
} else {
for _, cg := range children {
go z.updateCgAndWatch(cg)
}
}
<-eventChan
}
}
func (km *KMonitor) processFatalError() {
err := <-km.fatalErrorChan
logger.Fatal("%s", err.Error())
os.Exit(255)
}
func (td *TransDi) processConsumerError(cerr *sarama.ConsumerError) {
logger.Fatal("consumer error: topic=%s, partition=%d, %s", cerr.Topic, cerr.Partition, cerr.Error())
td.fatalErrorChan <- cerr
}