Beispiel #1
0
func (transfer *KTransfer) initModules() (err error) {
	transfer.modules = make(map[string]*Module)
	for _, moduleName := range transfer.moduleEnabled {
		module := &Module{
			name:                   moduleName,
			zkHosts:                transfer.zkHosts,
			zkChroot:               transfer.zkChroot,
			zkTimeout:              transfer.zkTimeout,
			zkFailRetryInterval:    transfer.zkFailRetryInterval,
			moduleConfDir:          transfer.moduleConfDir,
			fatalErrorChan:         transfer.fatalErrorChan,
			expectedProcessingTime: 100 * time.Millisecond,
			waitingQueueSize:       256,
			zkOffsetUpdateInterval: 10 * time.Second,
		}
		err = module.init()
		if err != nil {
			return
		}
		transfer.modules[moduleName] = module
		logger.Notice("succeeded in initing module [%s]", moduleName)
	}
	logger.Notice("all modules enabled have been inited")
	return nil
}
Beispiel #2
0
func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	startTime := time.Now()
	qv := r.URL.Query()
	id, partition, offset, err := h.bs.km.parseKey(qv.Get("key"))
	if err != nil {
		logger.Warning("fail to parseKey: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	val, err := h.getData(id, partition, offset)
	if err != nil {
		logger.Warning("fail to getData: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
	_, err = w.Write(val)
	if err != nil {
		logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	endTime := time.Now()
	costTimeUS := endTime.Sub(startTime) / time.Microsecond
	logger.Notice("success process get: %s, cost_us=%d, datalen=%d", r.URL.String(), costTimeUS, len(val))
	return
}
Beispiel #3
0
func (b *Broker) updateBrokerOffset() {
	if b.client == nil || b.client.Closed() {
		logger.Notice("broker.client not ready yet")
		return
	}
	topics, err := b.client.Topics()
	if err != nil {
		logger.Warning("fail to b.client.Topics(): %s", err.Error())
		return
	}
	for _, topic := range topics {
		partitions, err := b.client.Partitions(topic)
		if err != nil {
			logger.Warning("fail to b.client.Partitions(): %s", err.Error())
			continue
		}
		for _, partition := range partitions {
			offset, err := b.client.GetOffset(topic, partition, sarama.OffsetNewest)
			if err != nil {
				logger.Warning("fail to b.client.GetOffset topic=%s, partition=%d, err=%s", topic, partition, err.Error())
				continue
			}
			// TODO: use pool
			b.brokerOffsetChan <- &BrokerOffset{
				topic:     topic,
				partition: partition,
				offset:    offset,
			}
			logger.Debug("offset: topic=%s, partition=%d, offset=%d", topic, partition, offset)
		}
	}
}
func (hp *HttpProtocol) transData(server string, transData *TransData) (err error) {
	/* compose req */
	uri := hp.regTopic.ReplaceAllString(hp.uri, transData.topic)
	uri = hp.regMethod.ReplaceAllString(uri, transData.method)
	uri = hp.regPartition.ReplaceAllString(uri, strconv.FormatInt(int64(transData.partition), 10))
	uri = hp.regTransid.ReplaceAllString(uri, strconv.FormatInt(transData.transid, 10))
	url := fmt.Sprintf("http://%s%s", server, uri)
	req, err := http.NewRequest("POST", url, bytes.NewReader(transData.data))
	if err != nil {
		logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, err=%s", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, err.Error())
		return err
	}
	/* add header */
	req.Header = hp.header
	req.Host = hp.reqHeaderHost
	/* Post */
	res, err := hp.client.Do(req)
	if err != nil {
		logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, err=%s", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, err.Error())
		return err
	}
	defer res.Body.Close()
	/* check res: 200 不重试;其他,重试 */
	if res.StatusCode == http.StatusOK {
		logger.Notice("module [%s]: success transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, datalen=%d, http_status_code=%d", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, len(transData.data), res.StatusCode)
		return nil
	} else {
		logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, http_status_code=%d", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, res.StatusCode)
		return errors.New("fail to trans")
	}
	return nil
}
Beispiel #5
0
func (h *StoreHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	startTime := time.Now()
	/* check query */
	if r.ContentLength <= 0 {
		logger.Warning("invalid query, need post data: %s", r.URL.String())
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	// qv := r.URL.Query()
	sr := h.getStoreReq()
	defer h.putStoreReq(sr)
	nr, err := sr.reqBuffer.ReadFrom(r.Body)
	if int64(nr) != r.ContentLength || err != nil {
		logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	err = h.parseReq(sr)
	if err != nil {
		logger.Warning("invalid query, parseReq failed : %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	err = h.bs.store.addNewData(sr)
	if err != nil {
		logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)
	endTime := time.Now()
	costTimeUS := endTime.Sub(startTime) / time.Microsecond
	logger.Notice("success process store: %s, id=%d, cost_us=%d", r.URL.String(), sr.id, costTimeUS)
	return
}
Beispiel #6
0
func (rcp *RedisCachePool) init() (err error) {
	err = rcp.initRedisServers()
	if err != nil {
		return
	}
	logger.Notice("success init RedisPools for cluster [%s]", rcp.clusterName)
	return nil
}
Beispiel #7
0
func (d *Deleter) Run() (err error) {
	err = d.httpServer.ListenAndServe()
	if err != nil {
		logger.Warning("fail to httpServer.ListenAndServe: %s", err.Error())
		return
	}
	logger.Notice("httpServer.ListenAndServe end")
	return nil
}
Beispiel #8
0
func (cc *CacheCluster) init() (err error) {
	err = cc.initCachePool()
	if err != nil {
		return
	}
	err = cc.initMqClient()
	if err != nil {
		return
	}
	logger.Notice("success init CacheCluster: [%s]", cc.clusterName)
	return nil
}
Beispiel #9
0
func (transfer *KTransfer) Run() {
	logger.Notice("ktransfer start")
	for _, module := range transfer.modules {
		go module.run()
	}
	for {
		select {
		case err := <-transfer.fatalErrorChan:
			logger.Fatal("%s", err.Error())
			return
		}
	}
	return
}
Beispiel #10
0
func (worker *TransWorker) trans(transData *TransData) {
	var err error
	retry := 0
	for {
		serverId := (worker.lastServerId + 1) % (worker.serverNum)
		worker.lastServerId = serverId
		startTime := time.Now()
		err = worker.protocol.transData(worker.backendServers[serverId], transData)
		timeUsedMs := int(time.Now().Sub(startTime) / time.Millisecond)
		if err == nil {
			logger.Notice("module [%s]: worker[%d] success transData: server=%s, topic=%s, partition=%d, transid=%d, method=%s, proctime=%dms", worker.moduleName, worker.workerId, worker.backendServers[serverId], transData.topic, transData.partition, transData.transid, transData.method, timeUsedMs)
			ackData := &AckData{
				transid:  transData.transid,
				workerId: worker.workerId,
			}
			worker.ackDataChan <- ackData
			logger.Debug("add ackData: transid=%d", ackData.transid)
			return
		}
		logger.Notice("module [%s]: worker[%d] fail transData: server=%s, topic=%s, partition=%d, transid=%d, method=%s, proctime=%dms", worker.moduleName, worker.workerId, worker.backendServers[serverId], transData.topic, transData.partition, transData.transid, transData.method, timeUsedMs)
		/* check need retry */
		if worker.maxRetryTimes == -1 || retry < worker.maxRetryTimes {
			retry++
			time.Sleep(worker.failRetryInterval)
			continue
		}
		/* reatch retry limit */
		ackData := &AckData{
			transid:  transData.transid,
			workerId: worker.workerId,
		}
		worker.ackDataChan <- ackData
		logger.Debug("add ackData: transid=%d", ackData.transid)
		return
	}
}
Beispiel #11
0
func (ch *CmHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	startTime := time.Now()
	/* check query */
	if r.ContentLength <= 0 {
		logger.Warning("invalid query, need post data: %s", r.URL.String())
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	qv := r.URL.Query()
	post := make([]byte, r.ContentLength)
	nr, err := io.ReadFull(r.Body, post)
	if int64(nr) != r.ContentLength {
		logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	/* compose CmData */
	cmData := ch.cdp.fetch()
	defer ch.cdp.put(cmData)
	switch qv.Get("acks") {
	case "0":
		cmData.requiredAcks = sarama.NoResponse
	case "1":
		cmData.requiredAcks = sarama.WaitForLocal
	case "-1":
		cmData.requiredAcks = sarama.WaitForAll
	default:
		cmData.requiredAcks = sarama.WaitForLocal
	}
	cmData.topic = qv.Get("topic")
	cmData.key = qv.Get("key")
	cmData.data = post
	/* commit */
	ch.kp.producer.produce(cmData)
	/* wait res */
	<-cmData.cmDoneChan
	if cmData.err != nil {
		logger.Warning("fail to commit req: %s, error: %s", r.URL.String(), (*(cmData.err)).Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	endTime := time.Now()
	costTimeUS := endTime.Sub(startTime) / time.Microsecond
	// TODO
	logger.Notice("success process commit: %s, cost_us=%d, datalen=%d, offset=%d, partition=%d", r.URL.String(), costTimeUS, nr, cmData.offset, cmData.partition)
	w.WriteHeader(http.StatusOK)
	return
}
Beispiel #12
0
func (cc *CacheCluster) doDelFromMq(keys []interface{}, delTime, curDelay int64, doneServers map[string]interface{}, firstDel bool) (err error) {
	/* time to del ? */
	now := time.Now().Unix()
	delta := delTime + curDelay - now
	logger.Debug("delta=%d, delTime=%d, curDelay=%d, now=%d, firtDel=%t", delta, delTime, curDelay, now, firstDel)
	if delta > 0 {
		time.Sleep(time.Duration(delta) * time.Second)
		now = now + delta
	}
	err = cc.cachePool.doDel(keys, doneServers)
	if err != nil {
		/* put back to mq */
		err = cc.mqClient.addKeys(keys, doneServers, delTime, curDelay, false)
		if err != nil {
			return
		}
	} else {
		logger.Notice("del all keys done: cluster=%s, keys=%v, delay=%d", cc.clusterName, keys, curDelay)
	}
	if !firstDel {
		/* not first Del, so all delays has been in mq, we just quit and wait */
		return nil
	}
	/* ok, it is first del, we add all delays to mq here */
	ccc := cc.ccc
	delay := now - delTime
	logger.Debug("%d = %d - %d", delay, now, delTime)
	idx := sort.SearchInts(ccc.delayConfig, int(delay))
	if idx == len(ccc.delayConfig) {
		/* done */
		return nil
	}
	for ; idx < len(ccc.delayConfig); idx++ {
		if int64(ccc.delayConfig[idx]) <= delay {
			continue
		}
		// add back to mq with doneServers empty
		err = cc.mqClient.addKeys(keys, nil, delTime, int64(ccc.delayConfig[idx]), false)
		if err != nil {
			return
		}
	}
	return nil
}
Beispiel #13
0
func (td *TransDi) initTransWorkers() (err error) {
	for id := uint32(0); id < td.workerNum; id++ {
		worker := &TransWorker{
			moduleName:        td.moduleName,
			workerId:          id,
			backendServers:    td.backendServers,
			protocolConfig:    td.protocolConfig,
			fatalErrorChan:    td.fatalErrorChan,
			inWork:            false,
			ackDataChan:       td.ackDataChan,
			maxRetryTimes:     td.maxRetryTimes,
			failRetryInterval: td.failRetryInterval,
		}
		err = worker.init()
		if err != nil {
			return
		}
		td.transWorkers = append(td.transWorkers, worker)
	}
	logger.Notice("all workers have been inited for moudle [%s]", td.moduleName)
	return nil
}
Beispiel #14
0
func (httpServer *HttpServer) handleCm(w http.ResponseWriter, r *http.Request) {
	startTime := time.Now()
	/* check query */
	if r.ContentLength <= 0 {
		logger.Warning("invalid query, need post data: %s", r.URL.String())
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	qv := r.URL.Query()
	post := make([]byte, r.ContentLength)
	nr, err := io.ReadFull(r.Body, post)
	if int64(nr) != r.ContentLength {
		logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	/* compose CmData */
	cmData := httpServer.cdp.fetch()
	defer httpServer.cdp.put(cmData)
	cmData.topic = qv.Get("topic")
	cmData.key = qv.Get("key")
	cmData.data = post
	/* commit */
	httpServer.cmDataChan <- cmData
	/* wait res */
	<-cmData.cmDoneChan
	if cmData.err != nil {
		logger.Warning("fail to commit req: %s, error: %s", r.URL.String(), (*(cmData.err)).Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	endTime := time.Now()
	costTimeUS := endTime.Sub(startTime) / time.Microsecond
	// TODO
	logger.Notice("success process commit: %s, cost_us=%d, datalen=%d, offset=%d, partition=%d", r.URL.String(), costTimeUS, nr, cmData.offset, cmData.partition)
	w.WriteHeader(http.StatusOK)
	return
}
Beispiel #15
0
func (d *Deleter) init() (err error) {
	err = d.initConfig()
	if err != nil {
		return
	}
	err = d.initRuntimeConf()
	if err != nil {
		return
	}
	err = d.initLog()
	if err != nil {
		return
	}
	err = d.initCacheClusters()
	if err != nil {
		return
	}
	err = d.initServer()
	if err != nil {
		return
	}
	logger.Notice("deleter init success")
	return nil
}
Beispiel #16
0
func (h *AddHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	startTime := time.Now()
	/* check query */
	if r.ContentLength <= 0 {
		logger.Warning("invalid query, need post data: %s", r.URL.String())
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	// qv := r.URL.Query()
	bs := h.bs
	ad := bs.adp.fetch()
	defer bs.adp.put(ad)
	ad.buffer.Reset()
	nr, err := ad.buffer.ReadFrom(r.Body)
	if int64(nr) != r.ContentLength || err != nil {
		logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	ad.checksum()
	ad.Key = ""
	err = bs.dd.checkDup(ad)
	id := uint64(0)
	if err != nil {
		logger.Warning("fail to dd.checkDup: %s, %s", r.URL.String(), err.Error())
		// TODO: continue ?
	}
	if len(ad.Key) > 0 {
		// duplication
		// done
	} else {
		// new data
		id, err = bs.km.getNewId()
		if err != nil {
			logger.Warning("fail to getNewId: %s", err.Error())
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		p, o, err := bs.broker.addNewData(id, ad)
		if err != nil {
			logger.Warning("fail to addNewData: %s", err.Error())
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		key, err := bs.km.generateKey(id, p, o)
		if err != nil {
			logger.Warning("fail to generateKey: %s", err.Error())
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		ad.Key = key
		err = bs.dd.insertNew(ad)
		if err != nil {
			logger.Warning("fail to dd.insertNew: %s", err.Error())
			// only warning here
		}
	}
	// response ok
	w.WriteHeader(http.StatusOK)
	_, err = w.Write([]byte(ad.Key))
	if err != nil {
		logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error())
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	endTime := time.Now()
	costTimeUS := endTime.Sub(startTime) / time.Microsecond
	logger.Notice("success process add: %s, cost_us=%d, datalen=%d, id=%d, md5a=%d, md5b=%d, fnv1a32=%d, key=%s", r.URL.String(), costTimeUS, nr, id, ad.md5a, ad.md5b, ad.fnv1a32, ad.Key)
	return
}
Beispiel #17
0
func (m *Module) run() {
	logger.Notice("module [%s] start", m.name)
	m.transDi.run()
	// TODO: delete it later
	m.fatalErrorChan <- errors.New("no error")
}
Beispiel #18
0
func (d *Deleter) doDel(w http.ResponseWriter, r *http.Request) {
	/* get topic and method */
	logger.Debug("get one req: %s", r.URL.String())
	qv := r.URL.Query()
	topic := qv.Get("topic")
	if len(topic) == 0 {
		msg := fmt.Sprintf("invalid query, topic cannot be empty: %s", r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	cc, ok := d.cacheClusters[topic]
	if !ok {
		msg := fmt.Sprintf("invalid query, cache cluster is not exist: cluster [%s], %s", topic, r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}

	/* get post data */
	if r.ContentLength == 0 {
		logger.Warning("post data cannot be empty")
		d.response(w, http.StatusBadRequest, "post data cannot be empty")
		return
	}
	data := make([]byte, r.ContentLength)
	_, err := io.ReadFull(r.Body, data)
	if err != nil {
		msg := fmt.Sprintf("fail to read post data: %s", err.Error())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	/* unpack */
	buf := bytes.NewReader(data)
	buf.Seek(0, 0)
	msgr := msgp.NewReader(buf)
	reqi, err := msgr.ReadIntf()
	if err != nil {
		msg := fmt.Sprintf("fail to decode post data: %s", err.Error())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	req, ok := reqi.(map[string]interface{})
	if !ok {
		msg := fmt.Sprintf("invalid post data: data need map", r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	keysi, ok := req[gDelKeys]
	if !ok {
		msg := fmt.Sprintf("invalid post data: %s not exist: %s", gDelKeys, r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	keys, ok := keysi.([]interface{})
	if !ok {
		msg := fmt.Sprintf("invalid post data: %s not array: %s", gDelKeys, r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	if len(keys) == 0 {
		msg := fmt.Sprintf("invalid post data: len of %s array is 0: %s", gDelKeys, r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	delTimei, ok := req[gDelDelTime]
	if !ok {
		msg := fmt.Sprintf("invalid post data: %s not exist: %s", gDelDelTime, r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	delTime, ok := delTimei.(int64)
	if !ok {
		delTimet, ok := delTimei.(int)
		if !ok {
			delTimetu, ok := delTimei.(uint64)
			if !ok {
				msg := fmt.Sprintf("invalid post data: %s not int64 or uint64 or int: %s", gDelDelTime, r.URL.String())
				logger.Warning("%s", msg)
				d.response(w, http.StatusBadRequest, msg)
				return
			} else {
				delTime = int64(delTimetu)
			}
		} else {
			delTime = int64(delTimet)
		}
	}
	var curDelay int64
	curDelayi, ok := req[gDelCurDelay]
	if !ok {
		curDelay = 0
	} else if curDelay, ok = curDelayi.(int64); !ok {
		curDelayt, ok := curDelayi.(int)
		if !ok {
			curDelaytu, ok := curDelayi.(uint64)
			if !ok {
				msg := fmt.Sprintf("invalid post data: %s not int64 or int: %s", gDelCurDelay, r.URL.String())
				logger.Warning("%s", msg)
				d.response(w, http.StatusBadRequest, msg)
				return
			} else {
				curDelay = int64(curDelaytu)
			}
		} else {
			curDelay = int64(curDelayt)
		}
	}
	var isFromMq bool
	isFromMqi, ok := req[gDelFromMq]
	if !ok {
		isFromMq = false
	} else if isFromMq, ok = isFromMqi.(bool); !ok {
		msg := fmt.Sprintf("invalid post data: %s not bool: %s", gDelFromMq, r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	var doneServers map[string]interface{}
	doneServersi, ok := req[gDoneServers]
	if !ok {
		doneServers = make(map[string]interface{})
	} else if doneServers, ok = doneServersi.(map[string]interface{}); !ok {
		msg := fmt.Sprintf("invalid post data: %s not map[string]interface{}: %s", gDoneServers, r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}
	var firstDel bool
	firstDeli, ok := req[gFirstDel]
	if !ok {
		firstDel = true
	} else if firstDel, ok = firstDeli.(bool); !ok {
		msg := fmt.Sprintf("invalid post data: %s not bool: %s", gFirstDel, r.URL.String())
		logger.Warning("%s", msg)
		d.response(w, http.StatusBadRequest, msg)
		return
	}

	/* do del */
	err = cc.doDel(keys, delTime, curDelay, doneServers, isFromMq, firstDel)
	if err != nil {
		msg := fmt.Sprintf("fail to del: %s", err.Error())
		logger.Warning("%s", msg)
		d.response(w, http.StatusInternalServerError, msg)
		return
	}
	msg := fmt.Sprintf("success process: %s", r.URL.String())
	logger.Notice("%s", msg)
	d.response(w, http.StatusOK, msg)
	return
}
Beispiel #19
0
func (mc *MqClient) addKeys(keys []interface{}, doneServers map[string]interface{}, delTime, nextDelay int64, firstDel bool) (err error) {
	/* pack data */
	data := map[string]interface{}{
		gDelKeys:     keys,
		gDelDelTime:  delTime,
		gDelCurDelay: nextDelay,
		gDelFromMq:   true,
		gFirstDel:    firstDel,
		"method":     fmt.Sprintf("%s%d", gDelMethodPrefix, nextDelay), // ktransfer need it
	}
	if len(doneServers) > 0 {
		data[gDoneServers] = doneServers
	}
	var buf bytes.Buffer
	wr := msgp.NewWriter(&buf)
	err = wr.WriteIntf(data)
	if err != nil {
		logger.Warning("fail to msgp.WriteIntf: %s", err.Error())
		return
	}
	wr.Flush()
	r := bytes.NewReader(buf.Bytes())
	/* write mq */
	var mqServerAddrs []string
	if firstDel && len(mc.idcMqServerAddrs) > 0 {
		mqServerAddrs = mc.idcMqServerAddrs
	} else {
		mqServerAddrs = mc.delMqServerAddrs
	}
	toc := make(chan bool, 1)
	go mc.checkMqTimeout(toc)
	ns := len(mqServerAddrs)
	mc.lastServerId = (mc.lastServerId + 1) % ns
	i := 0
	for {
		r.Seek(0, 0)
		url := fmt.Sprintf("http://%s?topic=%s&method=%s%d", mqServerAddrs[mc.lastServerId], mc.clusterName, gDelMethodPrefix, nextDelay)
		req, err1 := http.NewRequest("POST", url, r)
		if err1 != nil {
			logger.Warning("fail to http.NewRequest: %s, %s", url, err1.Error())
			err = err1
			return
		}
		rsp, err1 := mc.client.Do(req)
		if err1 != nil {
			logger.Warning("fail to http.Client.Do: %s, %s", url, err1.Error())
			select {
			case <-toc:
				err = errors.New("add back to mq timeout")
				return
			default:
				/* do nothing */
			}
			i++
			if i == ns {
				err = errors.New("fail to add back to mq, all servers have been retried")
				return
			}
			mc.lastServerId = (mc.lastServerId + 1) % ns
			continue
		} else {
			rsp.Body.Close()
			logger.Notice("success add back to mq: %s", url)
			return nil
		}
	}
}