func (transfer *KTransfer) initModules() (err error) {
transfer.modules = make(map[string]*Module)
for _, moduleName := range transfer.moduleEnabled {
module := &Module{
name: moduleName,
zkHosts: transfer.zkHosts,
zkChroot: transfer.zkChroot,
zkTimeout: transfer.zkTimeout,
zkFailRetryInterval: transfer.zkFailRetryInterval,
moduleConfDir: transfer.moduleConfDir,
fatalErrorChan: transfer.fatalErrorChan,
expectedProcessingTime: 100 * time.Millisecond,
waitingQueueSize: 256,
zkOffsetUpdateInterval: 10 * time.Second,
}
err = module.init()
if err != nil {
return
}
transfer.modules[moduleName] = module
logger.Notice("succeeded in initing module [%s]", moduleName)
}
logger.Notice("all modules enabled have been inited")
return nil
}
func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
qv := r.URL.Query()
id, partition, offset, err := h.bs.km.parseKey(qv.Get("key"))
if err != nil {
logger.Warning("fail to parseKey: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusBadRequest)
return
}
val, err := h.getData(id, partition, offset)
if err != nil {
logger.Warning("fail to getData: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
_, err = w.Write(val)
if err != nil {
logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
endTime := time.Now()
costTimeUS := endTime.Sub(startTime) / time.Microsecond
logger.Notice("success process get: %s, cost_us=%d, datalen=%d", r.URL.String(), costTimeUS, len(val))
return
}
func (b *Broker) updateBrokerOffset() {
if b.client == nil || b.client.Closed() {
logger.Notice("broker.client not ready yet")
return
}
topics, err := b.client.Topics()
if err != nil {
logger.Warning("fail to b.client.Topics(): %s", err.Error())
return
}
for _, topic := range topics {
partitions, err := b.client.Partitions(topic)
if err != nil {
logger.Warning("fail to b.client.Partitions(): %s", err.Error())
continue
}
for _, partition := range partitions {
offset, err := b.client.GetOffset(topic, partition, sarama.OffsetNewest)
if err != nil {
logger.Warning("fail to b.client.GetOffset topic=%s, partition=%d, err=%s", topic, partition, err.Error())
continue
}
// TODO: use pool
b.brokerOffsetChan <- &BrokerOffset{
topic: topic,
partition: partition,
offset: offset,
}
logger.Debug("offset: topic=%s, partition=%d, offset=%d", topic, partition, offset)
}
}
}
func (hp *HttpProtocol) transData(server string, transData *TransData) (err error) {
/* compose req */
uri := hp.regTopic.ReplaceAllString(hp.uri, transData.topic)
uri = hp.regMethod.ReplaceAllString(uri, transData.method)
uri = hp.regPartition.ReplaceAllString(uri, strconv.FormatInt(int64(transData.partition), 10))
uri = hp.regTransid.ReplaceAllString(uri, strconv.FormatInt(transData.transid, 10))
url := fmt.Sprintf("http://%s%s", server, uri)
req, err := http.NewRequest("POST", url, bytes.NewReader(transData.data))
if err != nil {
logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, err=%s", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, err.Error())
return err
}
/* add header */
req.Header = hp.header
req.Host = hp.reqHeaderHost
/* Post */
res, err := hp.client.Do(req)
if err != nil {
logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, err=%s", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, err.Error())
return err
}
defer res.Body.Close()
/* check res: 200 不重试；其他，重试 */
if res.StatusCode == http.StatusOK {
logger.Notice("module [%s]: success transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, datalen=%d, http_status_code=%d", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, len(transData.data), res.StatusCode)
return nil
} else {
logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, http_status_code=%d", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, res.StatusCode)
return errors.New("fail to trans")
}
return nil
}
func (h *StoreHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
/* check query */
if r.ContentLength <= 0 {
logger.Warning("invalid query, need post data: %s", r.URL.String())
w.WriteHeader(http.StatusBadRequest)
return
}
// qv := r.URL.Query()
sr := h.getStoreReq()
defer h.putStoreReq(sr)
nr, err := sr.reqBuffer.ReadFrom(r.Body)
if int64(nr) != r.ContentLength || err != nil {
logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
err = h.parseReq(sr)
if err != nil {
logger.Warning("invalid query, parseReq failed : %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusBadRequest)
return
}
err = h.bs.store.addNewData(sr)
if err != nil {
logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
endTime := time.Now()
costTimeUS := endTime.Sub(startTime) / time.Microsecond
logger.Notice("success process store: %s, id=%d, cost_us=%d", r.URL.String(), sr.id, costTimeUS)
return
}
func (rcp *RedisCachePool) init() (err error) {
err = rcp.initRedisServers()
if err != nil {
return
}
logger.Notice("success init RedisPools for cluster [%s]", rcp.clusterName)
return nil
}
func (d *Deleter) Run() (err error) {
err = d.httpServer.ListenAndServe()
if err != nil {
logger.Warning("fail to httpServer.ListenAndServe: %s", err.Error())
return
}
logger.Notice("httpServer.ListenAndServe end")
return nil
}
func (cc *CacheCluster) init() (err error) {
err = cc.initCachePool()
if err != nil {
return
}
err = cc.initMqClient()
if err != nil {
return
}
logger.Notice("success init CacheCluster: [%s]", cc.clusterName)
return nil
}
func (transfer *KTransfer) Run() {
logger.Notice("ktransfer start")
for _, module := range transfer.modules {
go module.run()
}
for {
select {
case err := <-transfer.fatalErrorChan:
logger.Fatal("%s", err.Error())
return
}
}
return
}
func (worker *TransWorker) trans(transData *TransData) {
var err error
retry := 0
for {
serverId := (worker.lastServerId + 1) % (worker.serverNum)
worker.lastServerId = serverId
startTime := time.Now()
err = worker.protocol.transData(worker.backendServers[serverId], transData)
timeUsedMs := int(time.Now().Sub(startTime) / time.Millisecond)
if err == nil {
logger.Notice("module [%s]: worker[%d] success transData: server=%s, topic=%s, partition=%d, transid=%d, method=%s, proctime=%dms", worker.moduleName, worker.workerId, worker.backendServers[serverId], transData.topic, transData.partition, transData.transid, transData.method, timeUsedMs)
ackData := &AckData{
transid: transData.transid,
workerId: worker.workerId,
}
worker.ackDataChan <- ackData
logger.Debug("add ackData: transid=%d", ackData.transid)
return
}
logger.Notice("module [%s]: worker[%d] fail transData: server=%s, topic=%s, partition=%d, transid=%d, method=%s, proctime=%dms", worker.moduleName, worker.workerId, worker.backendServers[serverId], transData.topic, transData.partition, transData.transid, transData.method, timeUsedMs)
/* check need retry */
if worker.maxRetryTimes == -1 || retry < worker.maxRetryTimes {
retry++
time.Sleep(worker.failRetryInterval)
continue
}
/* reatch retry limit */
ackData := &AckData{
transid: transData.transid,
workerId: worker.workerId,
}
worker.ackDataChan <- ackData
logger.Debug("add ackData: transid=%d", ackData.transid)
return
}
}
func (ch *CmHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
/* check query */
if r.ContentLength <= 0 {
logger.Warning("invalid query, need post data: %s", r.URL.String())
w.WriteHeader(http.StatusBadRequest)
return
}
qv := r.URL.Query()
post := make([]byte, r.ContentLength)
nr, err := io.ReadFull(r.Body, post)
if int64(nr) != r.ContentLength {
logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
/* compose CmData */
cmData := ch.cdp.fetch()
defer ch.cdp.put(cmData)
switch qv.Get("acks") {
case "0":
cmData.requiredAcks = sarama.NoResponse
case "1":
cmData.requiredAcks = sarama.WaitForLocal
case "-1":
cmData.requiredAcks = sarama.WaitForAll
default:
cmData.requiredAcks = sarama.WaitForLocal
}
cmData.topic = qv.Get("topic")
cmData.key = qv.Get("key")
cmData.data = post
/* commit */
ch.kp.producer.produce(cmData)
/* wait res */
<-cmData.cmDoneChan
if cmData.err != nil {
logger.Warning("fail to commit req: %s, error: %s", r.URL.String(), (*(cmData.err)).Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
endTime := time.Now()
costTimeUS := endTime.Sub(startTime) / time.Microsecond
// TODO
logger.Notice("success process commit: %s, cost_us=%d, datalen=%d, offset=%d, partition=%d", r.URL.String(), costTimeUS, nr, cmData.offset, cmData.partition)
w.WriteHeader(http.StatusOK)
return
}
func (cc *CacheCluster) doDelFromMq(keys []interface{}, delTime, curDelay int64, doneServers map[string]interface{}, firstDel bool) (err error) {
/* time to del ? */
now := time.Now().Unix()
delta := delTime + curDelay - now
logger.Debug("delta=%d, delTime=%d, curDelay=%d, now=%d, firtDel=%t", delta, delTime, curDelay, now, firstDel)
if delta > 0 {
time.Sleep(time.Duration(delta) * time.Second)
now = now + delta
}
err = cc.cachePool.doDel(keys, doneServers)
if err != nil {
/* put back to mq */
err = cc.mqClient.addKeys(keys, doneServers, delTime, curDelay, false)
if err != nil {
return
}
} else {
logger.Notice("del all keys done: cluster=%s, keys=%v, delay=%d", cc.clusterName, keys, curDelay)
}
if !firstDel {
/* not first Del, so all delays has been in mq, we just quit and wait */
return nil
}
/* ok, it is first del, we add all delays to mq here */
ccc := cc.ccc
delay := now - delTime
logger.Debug("%d = %d - %d", delay, now, delTime)
idx := sort.SearchInts(ccc.delayConfig, int(delay))
if idx == len(ccc.delayConfig) {
/* done */
return nil
}
for ; idx < len(ccc.delayConfig); idx++ {
if int64(ccc.delayConfig[idx]) <= delay {
continue
}
// add back to mq with doneServers empty
err = cc.mqClient.addKeys(keys, nil, delTime, int64(ccc.delayConfig[idx]), false)
if err != nil {
return
}
}
return nil
}
func (td *TransDi) initTransWorkers() (err error) {
for id := uint32(0); id < td.workerNum; id++ {
worker := &TransWorker{
moduleName: td.moduleName,
workerId: id,
backendServers: td.backendServers,
protocolConfig: td.protocolConfig,
fatalErrorChan: td.fatalErrorChan,
inWork: false,
ackDataChan: td.ackDataChan,
maxRetryTimes: td.maxRetryTimes,
failRetryInterval: td.failRetryInterval,
}
err = worker.init()
if err != nil {
return
}
td.transWorkers = append(td.transWorkers, worker)
}
logger.Notice("all workers have been inited for moudle [%s]", td.moduleName)
return nil
}
func (httpServer *HttpServer) handleCm(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
/* check query */
if r.ContentLength <= 0 {
logger.Warning("invalid query, need post data: %s", r.URL.String())
w.WriteHeader(http.StatusBadRequest)
return
}
qv := r.URL.Query()
post := make([]byte, r.ContentLength)
nr, err := io.ReadFull(r.Body, post)
if int64(nr) != r.ContentLength {
logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
/* compose CmData */
cmData := httpServer.cdp.fetch()
defer httpServer.cdp.put(cmData)
cmData.topic = qv.Get("topic")
cmData.key = qv.Get("key")
cmData.data = post
/* commit */
httpServer.cmDataChan <- cmData
/* wait res */
<-cmData.cmDoneChan
if cmData.err != nil {
logger.Warning("fail to commit req: %s, error: %s", r.URL.String(), (*(cmData.err)).Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
endTime := time.Now()
costTimeUS := endTime.Sub(startTime) / time.Microsecond
// TODO
logger.Notice("success process commit: %s, cost_us=%d, datalen=%d, offset=%d, partition=%d", r.URL.String(), costTimeUS, nr, cmData.offset, cmData.partition)
w.WriteHeader(http.StatusOK)
return
}
func (d *Deleter) init() (err error) {
err = d.initConfig()
if err != nil {
return
}
err = d.initRuntimeConf()
if err != nil {
return
}
err = d.initLog()
if err != nil {
return
}
err = d.initCacheClusters()
if err != nil {
return
}
err = d.initServer()
if err != nil {
return
}
logger.Notice("deleter init success")
return nil
}
func (h *AddHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
/* check query */
if r.ContentLength <= 0 {
logger.Warning("invalid query, need post data: %s", r.URL.String())
w.WriteHeader(http.StatusBadRequest)
return
}
// qv := r.URL.Query()
bs := h.bs
ad := bs.adp.fetch()
defer bs.adp.put(ad)
ad.buffer.Reset()
nr, err := ad.buffer.ReadFrom(r.Body)
if int64(nr) != r.ContentLength || err != nil {
logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
ad.checksum()
ad.Key = ""
err = bs.dd.checkDup(ad)
id := uint64(0)
if err != nil {
logger.Warning("fail to dd.checkDup: %s, %s", r.URL.String(), err.Error())
// TODO: continue ?
}
if len(ad.Key) > 0 {
// duplication
// done
} else {
// new data
id, err = bs.km.getNewId()
if err != nil {
logger.Warning("fail to getNewId: %s", err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
p, o, err := bs.broker.addNewData(id, ad)
if err != nil {
logger.Warning("fail to addNewData: %s", err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
key, err := bs.km.generateKey(id, p, o)
if err != nil {
logger.Warning("fail to generateKey: %s", err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
ad.Key = key
err = bs.dd.insertNew(ad)
if err != nil {
logger.Warning("fail to dd.insertNew: %s", err.Error())
// only warning here
}
}
// response ok
w.WriteHeader(http.StatusOK)
_, err = w.Write([]byte(ad.Key))
if err != nil {
logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
endTime := time.Now()
costTimeUS := endTime.Sub(startTime) / time.Microsecond
logger.Notice("success process add: %s, cost_us=%d, datalen=%d, id=%d, md5a=%d, md5b=%d, fnv1a32=%d, key=%s", r.URL.String(), costTimeUS, nr, id, ad.md5a, ad.md5b, ad.fnv1a32, ad.Key)
return
}
func (m *Module) run() {
logger.Notice("module [%s] start", m.name)
m.transDi.run()
// TODO: delete it later
m.fatalErrorChan <- errors.New("no error")
}
func (d *Deleter) doDel(w http.ResponseWriter, r *http.Request) {
/* get topic and method */
logger.Debug("get one req: %s", r.URL.String())
qv := r.URL.Query()
topic := qv.Get("topic")
if len(topic) == 0 {
msg := fmt.Sprintf("invalid query, topic cannot be empty: %s", r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
cc, ok := d.cacheClusters[topic]
if !ok {
msg := fmt.Sprintf("invalid query, cache cluster is not exist: cluster [%s], %s", topic, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
/* get post data */
if r.ContentLength == 0 {
logger.Warning("post data cannot be empty")
d.response(w, http.StatusBadRequest, "post data cannot be empty")
return
}
data := make([]byte, r.ContentLength)
_, err := io.ReadFull(r.Body, data)
if err != nil {
msg := fmt.Sprintf("fail to read post data: %s", err.Error())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
/* unpack */
buf := bytes.NewReader(data)
buf.Seek(0, 0)
msgr := msgp.NewReader(buf)
reqi, err := msgr.ReadIntf()
if err != nil {
msg := fmt.Sprintf("fail to decode post data: %s", err.Error())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
req, ok := reqi.(map[string]interface{})
if !ok {
msg := fmt.Sprintf("invalid post data: data need map", r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
keysi, ok := req[gDelKeys]
if !ok {
msg := fmt.Sprintf("invalid post data: %s not exist: %s", gDelKeys, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
keys, ok := keysi.([]interface{})
if !ok {
msg := fmt.Sprintf("invalid post data: %s not array: %s", gDelKeys, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
if len(keys) == 0 {
msg := fmt.Sprintf("invalid post data: len of %s array is 0: %s", gDelKeys, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
delTimei, ok := req[gDelDelTime]
if !ok {
msg := fmt.Sprintf("invalid post data: %s not exist: %s", gDelDelTime, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
delTime, ok := delTimei.(int64)
if !ok {
delTimet, ok := delTimei.(int)
if !ok {
delTimetu, ok := delTimei.(uint64)
if !ok {
msg := fmt.Sprintf("invalid post data: %s not int64 or uint64 or int: %s", gDelDelTime, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
} else {
delTime = int64(delTimetu)
}
} else {
delTime = int64(delTimet)
}
}
var curDelay int64
curDelayi, ok := req[gDelCurDelay]
if !ok {
curDelay = 0
} else if curDelay, ok = curDelayi.(int64); !ok {
curDelayt, ok := curDelayi.(int)
if !ok {
curDelaytu, ok := curDelayi.(uint64)
if !ok {
msg := fmt.Sprintf("invalid post data: %s not int64 or int: %s", gDelCurDelay, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
} else {
curDelay = int64(curDelaytu)
}
} else {
curDelay = int64(curDelayt)
}
}
var isFromMq bool
isFromMqi, ok := req[gDelFromMq]
if !ok {
isFromMq = false
} else if isFromMq, ok = isFromMqi.(bool); !ok {
msg := fmt.Sprintf("invalid post data: %s not bool: %s", gDelFromMq, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
var doneServers map[string]interface{}
doneServersi, ok := req[gDoneServers]
if !ok {
doneServers = make(map[string]interface{})
} else if doneServers, ok = doneServersi.(map[string]interface{}); !ok {
msg := fmt.Sprintf("invalid post data: %s not map[string]interface{}: %s", gDoneServers, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
var firstDel bool
firstDeli, ok := req[gFirstDel]
if !ok {
firstDel = true
} else if firstDel, ok = firstDeli.(bool); !ok {
msg := fmt.Sprintf("invalid post data: %s not bool: %s", gFirstDel, r.URL.String())
logger.Warning("%s", msg)
d.response(w, http.StatusBadRequest, msg)
return
}
/* do del */
err = cc.doDel(keys, delTime, curDelay, doneServers, isFromMq, firstDel)
if err != nil {
msg := fmt.Sprintf("fail to del: %s", err.Error())
logger.Warning("%s", msg)
d.response(w, http.StatusInternalServerError, msg)
return
}
msg := fmt.Sprintf("success process: %s", r.URL.String())
logger.Notice("%s", msg)
d.response(w, http.StatusOK, msg)
return
}
func (mc *MqClient) addKeys(keys []interface{}, doneServers map[string]interface{}, delTime, nextDelay int64, firstDel bool) (err error) {
/* pack data */
data := map[string]interface{}{
gDelKeys: keys,
gDelDelTime: delTime,
gDelCurDelay: nextDelay,
gDelFromMq: true,
gFirstDel: firstDel,
"method": fmt.Sprintf("%s%d", gDelMethodPrefix, nextDelay), // ktransfer need it
}
if len(doneServers) > 0 {
data[gDoneServers] = doneServers
}
var buf bytes.Buffer
wr := msgp.NewWriter(&buf)
err = wr.WriteIntf(data)
if err != nil {
logger.Warning("fail to msgp.WriteIntf: %s", err.Error())
return
}
wr.Flush()
r := bytes.NewReader(buf.Bytes())
/* write mq */
var mqServerAddrs []string
if firstDel && len(mc.idcMqServerAddrs) > 0 {
mqServerAddrs = mc.idcMqServerAddrs
} else {
mqServerAddrs = mc.delMqServerAddrs
}
toc := make(chan bool, 1)
go mc.checkMqTimeout(toc)
ns := len(mqServerAddrs)
mc.lastServerId = (mc.lastServerId + 1) % ns
i := 0
for {
r.Seek(0, 0)
url := fmt.Sprintf("http://%s?topic=%s&method=%s%d", mqServerAddrs[mc.lastServerId], mc.clusterName, gDelMethodPrefix, nextDelay)
req, err1 := http.NewRequest("POST", url, r)
if err1 != nil {
logger.Warning("fail to http.NewRequest: %s, %s", url, err1.Error())
err = err1
return
}
rsp, err1 := mc.client.Do(req)
if err1 != nil {
logger.Warning("fail to http.Client.Do: %s, %s", url, err1.Error())
select {
case <-toc:
err = errors.New("add back to mq timeout")
return
default:
/* do nothing */
}
i++
if i == ns {
err = errors.New("fail to add back to mq, all servers have been retried")
return
}
mc.lastServerId = (mc.lastServerId + 1) % ns
continue
} else {
rsp.Body.Close()
logger.Notice("success add back to mq: %s", url)
return nil
}
}
}