/* do not change state in this func */
func (td *TransDi) processAckData(ackData *AckData) {
transid := ackData.transid
workerId := ackData.workerId
logger.Debug("process ack: transid=%d, workerId=%d", transid, workerId)
/* narrow window */
msg := td.transWindow.ackOne(transid)
if msg != nil && msg.Offset > td.ackedMinTransid {
td.ackedMinTransid = msg.Offset
err := td.cg.CommitUpto(msg)
if err != nil {
// TODO: optimized
logger.Warning("fail to consumergroup.CommitUpto(): %s", err.Error())
td.fatalErrorChan <- err
return
}
logger.Debug("consumergroup.CommitUpTo %d", msg.Offset)
}
logger.Debug("transWindow size: %d", td.transWindow.window.Len())
/* move worker */
worker := td.transWorkers[workerId]
worker.inWork = false
err := worker.workIfNeed()
if err != nil {
logger.Fatal("fail to let worker to work: %s", err.Error())
td.fatalErrorChan <- err
return
}
}
func (z *Zookeeper) updateBrokerListAndWatch() {
for {
children, _, eventChan, err := z.zkConn.ChildrenW(fmt.Sprintf("%s/brokers/ids", z.cc.chroot))
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.ChildrenW: %s", err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
if len(children) == 0 {
logger.Warning("no broker found")
} else {
logger.Debug("using child: %s", children[0])
val, _, err := z.zkConn.Get(fmt.Sprintf("%s/brokers/ids/%s", z.cc.chroot, children[0]))
var brokerIdVal BrokerIdVal
err = json.Unmarshal(val, &brokerIdVal)
if err != nil {
logger.Warning("fail to json.Unmarshal for broker id %s: %s", children[0], err.Error())
}
logger.Debug("broker id %s: host [%s] port [%d]", children[0], brokerIdVal.Host, brokerIdVal.Port)
z.brokerIdValChan <- &brokerIdVal
}
<-eventChan
}
}
func (worker *TransWorker) run() {
logger.Debug("worker %d start for module [%s]", worker.workerId, worker.moduleName)
for transData := range worker.transDataChan {
logger.Debug("worker begin: transid=%d, topic=%s, method=%s", transData.transid, transData.topic, transData.method)
/* will block in worker.trans untile succeed in trans or reach retry limit */
worker.trans(transData)
}
}
func (b *Broker) updateBrokerOffset() {
if b.client == nil || b.client.Closed() {
logger.Notice("broker.client not ready yet")
return
}
topics, err := b.client.Topics()
if err != nil {
logger.Warning("fail to b.client.Topics(): %s", err.Error())
return
}
for _, topic := range topics {
partitions, err := b.client.Partitions(topic)
if err != nil {
logger.Warning("fail to b.client.Partitions(): %s", err.Error())
continue
}
for _, partition := range partitions {
offset, err := b.client.GetOffset(topic, partition, sarama.OffsetNewest)
if err != nil {
logger.Warning("fail to b.client.GetOffset topic=%s, partition=%d, err=%s", topic, partition, err.Error())
continue
}
// TODO: use pool
b.brokerOffsetChan <- &BrokerOffset{
topic: topic,
partition: partition,
offset: offset,
}
logger.Debug("offset: topic=%s, partition=%d, offset=%d", topic, partition, offset)
}
}
}
func (td *TransDi) run() {
logger.Debug("transdi start for module [%s]", td.moduleName)
for _, worker := range td.transWorkers {
go worker.run()
}
td.state = WAIT_ACK_NONBLOCK
for {
switch td.state {
case WAIT_ACK_NONBLOCK:
td.waitAckNonBlock()
continue
case CHECK_NEED_WAIT_ACK:
td.checkNeedWaitAck()
continue
case WAIT_ACK_BLOCK:
td.waitAckBlock()
continue
case WAIT_ACK_AND_DI:
td.waitAckAndDi()
continue
default:
td.fatalErrorChan <- errors.New(fmt.Sprintf("invalid transdi run state: %d", td.state))
break
}
}
}
func (cc *CacheCluster) doDelFromMq(keys []interface{}, delTime, curDelay int64, doneServers map[string]interface{}, firstDel bool) (err error) {
/* time to del ? */
now := time.Now().Unix()
delta := delTime + curDelay - now
logger.Debug("delta=%d, delTime=%d, curDelay=%d, now=%d, firtDel=%t", delta, delTime, curDelay, now, firstDel)
if delta > 0 {
time.Sleep(time.Duration(delta) * time.Second)
now = now + delta
}
err = cc.cachePool.doDel(keys, doneServers)
if err != nil {
/* put back to mq */
err = cc.mqClient.addKeys(keys, doneServers, delTime, curDelay, false)
if err != nil {
return
}
} else {
logger.Notice("del all keys done: cluster=%s, keys=%v, delay=%d", cc.clusterName, keys, curDelay)
}
if !firstDel {
/* not first Del, so all delays has been in mq, we just quit and wait */
return nil
}
/* ok, it is first del, we add all delays to mq here */
ccc := cc.ccc
delay := now - delTime
logger.Debug("%d = %d - %d", delay, now, delTime)
idx := sort.SearchInts(ccc.delayConfig, int(delay))
if idx == len(ccc.delayConfig) {
/* done */
return nil
}
for ; idx < len(ccc.delayConfig); idx++ {
if int64(ccc.delayConfig[idx]) <= delay {
continue
}
// add back to mq with doneServers empty
err = cc.mqClient.addKeys(keys, nil, delTime, int64(ccc.delayConfig[idx]), false)
if err != nil {
return
}
}
return nil
}
func (hp *HttpProtocol) init() (err error) {
logger.Debug("http protocol init")
err = hp.initConfig()
if err != nil {
return
}
err = hp.initRegs()
if err != nil {
return
}
return nil
}
func (worker *TransWorker) init() (err error) {
logger.Debug("init trans worker for module [%s]: %d", worker.moduleName, worker.workerId)
err = worker.initProtocol()
if err != nil {
return
}
worker.transQueue = list.New()
worker.transDataChan = make(chan *TransData)
worker.inWork = false
worker.lastServerId = -1
worker.serverNum = len(worker.backendServers)
return nil
}
/* Notice: this func shall ONLY be called in transdi routine */
func (worker *TransWorker) workIfNeed() (err error) {
if !worker.inWork {
e := worker.transQueue.Front()
if e == nil {
return nil
}
transData := e.Value.(*TransData)
logger.Debug("add to worker chan: transid=%d, topic=%s, method=%s", transData.transid, transData.topic, transData.method)
worker.transDataChan <- transData
worker.inWork = true
worker.transQueue.Remove(e)
}
return nil
}
func (b *Broker) updateBrokerClient(brokerIdVal *BrokerIdVal) {
if b.client != nil && b.client.Closed() == false {
b.client.Close()
}
b.client = nil
var err error
b.client, err = sarama.NewClient([]string{fmt.Sprintf("%s:%d", brokerIdVal.Host, brokerIdVal.Port)}, b.brokerConfig)
if err != nil {
err = errors.New(fmt.Sprintf("fail to sarama.NewClient: %s", err.Error()))
logger.Fatal(err.Error())
b.fatalErrorChan <- err
return
}
logger.Debug("success update broker client: %s:%d", brokerIdVal.Host, brokerIdVal.Port)
}
func (worker *TransWorker) trans(transData *TransData) {
var err error
retry := 0
for {
serverId := (worker.lastServerId + 1) % (worker.serverNum)
worker.lastServerId = serverId
startTime := time.Now()
err = worker.protocol.transData(worker.backendServers[serverId], transData)
timeUsedMs := int(time.Now().Sub(startTime) / time.Millisecond)
if err == nil {
logger.Notice("module [%s]: worker[%d] success transData: server=%s, topic=%s, partition=%d, transid=%d, method=%s, proctime=%dms", worker.moduleName, worker.workerId, worker.backendServers[serverId], transData.topic, transData.partition, transData.transid, transData.method, timeUsedMs)
ackData := &AckData{
transid: transData.transid,
workerId: worker.workerId,
}
worker.ackDataChan <- ackData
logger.Debug("add ackData: transid=%d", ackData.transid)
return
}
logger.Notice("module [%s]: worker[%d] fail transData: server=%s, topic=%s, partition=%d, transid=%d, method=%s, proctime=%dms", worker.moduleName, worker.workerId, worker.backendServers[serverId], transData.topic, transData.partition, transData.transid, transData.method, timeUsedMs)
/* check need retry */
if worker.maxRetryTimes == -1 || retry < worker.maxRetryTimes {
retry++
time.Sleep(worker.failRetryInterval)
continue
}
/* reatch retry limit */
ackData := &AckData{
transid: transData.transid,
workerId: worker.workerId,
}
worker.ackDataChan <- ackData
logger.Debug("add ackData: transid=%d", ackData.transid)
return
}
}
func (z *Zookeeper) updatePartitionAndWatch(cg, topic string, partition int32) {
path := fmt.Sprintf("%s/consumers/%s/offsets/%s/%d", z.cc.chroot, cg, topic, partition)
for {
val, _, eventChan, err := z.zkConn.GetW(path)
if err == zk.ErrNoNode {
_, _, eventChan, err = z.zkConn.ExistsW(path)
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.ExistsW: path=%s, %s", path, err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
<-eventChan
continue
}
if err != nil {
err = errors.New(fmt.Sprintf("fail to zk.GetW: path=%s, %s", path, err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
if len(val) == 0 {
logger.Warning("transfer partition val is empty: path=%s", path)
} else {
offset, err := strconv.ParseInt(string(val), 10, 64)
if err != nil {
err = errors.New(fmt.Sprintf("fail to transfer: strconv.ParseInt: %s", err.Error()))
logger.Fatal(err.Error())
z.fatalErrorChan <- err
return
}
z.transferOffsetChan <- &TransferOffset{
cg: cg,
topic: topic,
partition: partition,
offset: offset,
}
logger.Debug("see transfer: cg=%s, topic=%s, partition=%d, offset=%d", cg, topic, partition, offset)
}
event := <-eventChan
// let root watcher to re-init
if event.Type == zk.EventNotWatching {
return
}
}
}
func (d *Deleter) doDel(w http.ResponseWriter, r *http.Request) {
/* get topic and method */
logger.Debug("get one req: %s", r.URL.String())
qv := r.URL.Query()
topic := qv.Get("topic")
if len(topic) == 0 {
msg := fmt.Sprintf("invalid query, topic cannot be empty: %s", r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
cc, ok := d.cacheClusters[topic]
if !ok {
msg := fmt.Sprintf("invalid query, cache cluster is not exist: cluster [%s], %s", topic, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
/* get post data */
if r.ContentLength == 0 {
logger.Warning("post data cannot be empty")
d.response(w, http.StatusBadRequest, "post data cannot be empty")
return
}
data := make([]byte, r.ContentLength)
_, err := io.ReadFull(r.Body, data)
if err != nil {
msg := fmt.Sprintf("fail to read post data: %s", err.Error())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
/* unpack */
buf := bytes.NewReader(data)
buf.Seek(0, 0)
msgr := msgp.NewReader(buf)
reqi, err := msgr.ReadIntf()
if err != nil {
msg := fmt.Sprintf("fail to decode post data: %s", err.Error())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
req, ok := reqi.(map[string]interface{})
if !ok {
msg := fmt.Sprintf("invalid post data: data need map", r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
keysi, ok := req[gDelKeys]
if !ok {
msg := fmt.Sprintf("invalid post data: %s not exist: %s", gDelKeys, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
keys, ok := keysi.([]interface{})
if !ok {
msg := fmt.Sprintf("invalid post data: %s not array: %s", gDelKeys, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
if len(keys) == 0 {
msg := fmt.Sprintf("invalid post data: len of %s array is 0: %s", gDelKeys, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
delTimei, ok := req[gDelDelTime]
if !ok {
msg := fmt.Sprintf("invalid post data: %s not exist: %s", gDelDelTime, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
delTime, ok := delTimei.(int64)
if !ok {
delTimet, ok := delTimei.(int)
if !ok {
delTimetu, ok := delTimei.(uint64)
if !ok {
msg := fmt.Sprintf("invalid post data: %s not int64 or uint64 or int: %s", gDelDelTime, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
} else {
delTime = int64(delTimetu)
}
} else {
delTime = int64(delTimet)
}
}
var curDelay int64
curDelayi, ok := req[gDelCurDelay]
if !ok {
curDelay = 0
} else if curDelay, ok = curDelayi.(int64); !ok {
curDelayt, ok := curDelayi.(int)
if !ok {
curDelaytu, ok := curDelayi.(uint64)
if !ok {
msg := fmt.Sprintf("invalid post data: %s not int64 or int: %s", gDelCurDelay, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
} else {
curDelay = int64(curDelaytu)
}
} else {
curDelay = int64(curDelayt)
}
}
var isFromMq bool
isFromMqi, ok := req[gDelFromMq]
if !ok {
isFromMq = false
} else if isFromMq, ok = isFromMqi.(bool); !ok {
msg := fmt.Sprintf("invalid post data: %s not bool: %s", gDelFromMq, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
var doneServers map[string]interface{}
doneServersi, ok := req[gDoneServers]
if !ok {
doneServers = make(map[string]interface{})
} else if doneServers, ok = doneServersi.(map[string]interface{}); !ok {
msg := fmt.Sprintf("invalid post data: %s not map[string]interface{}: %s", gDoneServers, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
var firstDel bool
firstDeli, ok := req[gFirstDel]
if !ok {
firstDel = true
} else if firstDel, ok = firstDeli.(bool); !ok {
msg := fmt.Sprintf("invalid post data: %s not bool: %s", gFirstDel, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
/* do del */
err = cc.doDel(keys, delTime, curDelay, doneServers, isFromMq, firstDel)
if err != nil {
msg := fmt.Sprintf("fail to del: %s", err.Error())
logger.Warning("%s", msg)
d.response(w, http.StatusInternalServerError, msg)
return
}
msg := fmt.Sprintf("success process: %s", r.URL.String())
logger.Notice("%s", msg)
d.response(w, http.StatusOK, msg)
return
}