func (transfer *KTransfer) initModules() (err error) { transfer.modules = make(map[string]*Module) for _, moduleName := range transfer.moduleEnabled { module := &Module{ name: moduleName, zkHosts: transfer.zkHosts, zkChroot: transfer.zkChroot, zkTimeout: transfer.zkTimeout, zkFailRetryInterval: transfer.zkFailRetryInterval, moduleConfDir: transfer.moduleConfDir, fatalErrorChan: transfer.fatalErrorChan, expectedProcessingTime: 100 * time.Millisecond, waitingQueueSize: 256, zkOffsetUpdateInterval: 10 * time.Second, } err = module.init() if err != nil { return } transfer.modules[moduleName] = module logger.Notice("succeeded in initing module [%s]", moduleName) } logger.Notice("all modules enabled have been inited") return nil }
func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { startTime := time.Now() qv := r.URL.Query() id, partition, offset, err := h.bs.km.parseKey(qv.Get("key")) if err != nil { logger.Warning("fail to parseKey: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusBadRequest) return } val, err := h.getData(id, partition, offset) if err != nil { logger.Warning("fail to getData: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusInternalServerError) return } w.WriteHeader(http.StatusOK) _, err = w.Write(val) if err != nil { logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusInternalServerError) return } endTime := time.Now() costTimeUS := endTime.Sub(startTime) / time.Microsecond logger.Notice("success process get: %s, cost_us=%d, datalen=%d", r.URL.String(), costTimeUS, len(val)) return }
func (b *Broker) updateBrokerOffset() { if b.client == nil || b.client.Closed() { logger.Notice("broker.client not ready yet") return } topics, err := b.client.Topics() if err != nil { logger.Warning("fail to b.client.Topics(): %s", err.Error()) return } for _, topic := range topics { partitions, err := b.client.Partitions(topic) if err != nil { logger.Warning("fail to b.client.Partitions(): %s", err.Error()) continue } for _, partition := range partitions { offset, err := b.client.GetOffset(topic, partition, sarama.OffsetNewest) if err != nil { logger.Warning("fail to b.client.GetOffset topic=%s, partition=%d, err=%s", topic, partition, err.Error()) continue } // TODO: use pool b.brokerOffsetChan <- &BrokerOffset{ topic: topic, partition: partition, offset: offset, } logger.Debug("offset: topic=%s, partition=%d, offset=%d", topic, partition, offset) } } }
func (hp *HttpProtocol) transData(server string, transData *TransData) (err error) { /* compose req */ uri := hp.regTopic.ReplaceAllString(hp.uri, transData.topic) uri = hp.regMethod.ReplaceAllString(uri, transData.method) uri = hp.regPartition.ReplaceAllString(uri, strconv.FormatInt(int64(transData.partition), 10)) uri = hp.regTransid.ReplaceAllString(uri, strconv.FormatInt(transData.transid, 10)) url := fmt.Sprintf("http://%s%s", server, uri) req, err := http.NewRequest("POST", url, bytes.NewReader(transData.data)) if err != nil { logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, err=%s", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, err.Error()) return err } /* add header */ req.Header = hp.header req.Host = hp.reqHeaderHost /* Post */ res, err := hp.client.Do(req) if err != nil { logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, err=%s", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, err.Error()) return err } defer res.Body.Close() /* check res: 200 不重试;其他,重试 */ if res.StatusCode == http.StatusOK { logger.Notice("module [%s]: success transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, datalen=%d, http_status_code=%d", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, len(transData.data), res.StatusCode) return nil } else { logger.Warning("module [%s]: fail to transData: url=%s, topic=%s, partition=%d, transid=%d, method=%s, http_status_code=%d", hp.moduleName, url, transData.topic, transData.partition, transData.transid, transData.method, res.StatusCode) return errors.New("fail to trans") } return nil }
func (h *StoreHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { startTime := time.Now() /* check query */ if r.ContentLength <= 0 { logger.Warning("invalid query, need post data: %s", r.URL.String()) w.WriteHeader(http.StatusBadRequest) return } // qv := r.URL.Query() sr := h.getStoreReq() defer h.putStoreReq(sr) nr, err := sr.reqBuffer.ReadFrom(r.Body) if int64(nr) != r.ContentLength || err != nil { logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusInternalServerError) return } err = h.parseReq(sr) if err != nil { logger.Warning("invalid query, parseReq failed : %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusBadRequest) return } err = h.bs.store.addNewData(sr) if err != nil { logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusInternalServerError) return } w.WriteHeader(http.StatusOK) endTime := time.Now() costTimeUS := endTime.Sub(startTime) / time.Microsecond logger.Notice("success process store: %s, id=%d, cost_us=%d", r.URL.String(), sr.id, costTimeUS) return }
func (rcp *RedisCachePool) init() (err error) { err = rcp.initRedisServers() if err != nil { return } logger.Notice("success init RedisPools for cluster [%s]", rcp.clusterName) return nil }
func (d *Deleter) Run() (err error) { err = d.httpServer.ListenAndServe() if err != nil { logger.Warning("fail to httpServer.ListenAndServe: %s", err.Error()) return } logger.Notice("httpServer.ListenAndServe end") return nil }
func (cc *CacheCluster) init() (err error) { err = cc.initCachePool() if err != nil { return } err = cc.initMqClient() if err != nil { return } logger.Notice("success init CacheCluster: [%s]", cc.clusterName) return nil }
func (transfer *KTransfer) Run() { logger.Notice("ktransfer start") for _, module := range transfer.modules { go module.run() } for { select { case err := <-transfer.fatalErrorChan: logger.Fatal("%s", err.Error()) return } } return }
func (worker *TransWorker) trans(transData *TransData) { var err error retry := 0 for { serverId := (worker.lastServerId + 1) % (worker.serverNum) worker.lastServerId = serverId startTime := time.Now() err = worker.protocol.transData(worker.backendServers[serverId], transData) timeUsedMs := int(time.Now().Sub(startTime) / time.Millisecond) if err == nil { logger.Notice("module [%s]: worker[%d] success transData: server=%s, topic=%s, partition=%d, transid=%d, method=%s, proctime=%dms", worker.moduleName, worker.workerId, worker.backendServers[serverId], transData.topic, transData.partition, transData.transid, transData.method, timeUsedMs) ackData := &AckData{ transid: transData.transid, workerId: worker.workerId, } worker.ackDataChan <- ackData logger.Debug("add ackData: transid=%d", ackData.transid) return } logger.Notice("module [%s]: worker[%d] fail transData: server=%s, topic=%s, partition=%d, transid=%d, method=%s, proctime=%dms", worker.moduleName, worker.workerId, worker.backendServers[serverId], transData.topic, transData.partition, transData.transid, transData.method, timeUsedMs) /* check need retry */ if worker.maxRetryTimes == -1 || retry < worker.maxRetryTimes { retry++ time.Sleep(worker.failRetryInterval) continue } /* reatch retry limit */ ackData := &AckData{ transid: transData.transid, workerId: worker.workerId, } worker.ackDataChan <- ackData logger.Debug("add ackData: transid=%d", ackData.transid) return } }
func (ch *CmHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { startTime := time.Now() /* check query */ if r.ContentLength <= 0 { logger.Warning("invalid query, need post data: %s", r.URL.String()) w.WriteHeader(http.StatusBadRequest) return } qv := r.URL.Query() post := make([]byte, r.ContentLength) nr, err := io.ReadFull(r.Body, post) if int64(nr) != r.ContentLength { logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusInternalServerError) return } /* compose CmData */ cmData := ch.cdp.fetch() defer ch.cdp.put(cmData) switch qv.Get("acks") { case "0": cmData.requiredAcks = sarama.NoResponse case "1": cmData.requiredAcks = sarama.WaitForLocal case "-1": cmData.requiredAcks = sarama.WaitForAll default: cmData.requiredAcks = sarama.WaitForLocal } cmData.topic = qv.Get("topic") cmData.key = qv.Get("key") cmData.data = post /* commit */ ch.kp.producer.produce(cmData) /* wait res */ <-cmData.cmDoneChan if cmData.err != nil { logger.Warning("fail to commit req: %s, error: %s", r.URL.String(), (*(cmData.err)).Error()) w.WriteHeader(http.StatusInternalServerError) return } endTime := time.Now() costTimeUS := endTime.Sub(startTime) / time.Microsecond // TODO logger.Notice("success process commit: %s, cost_us=%d, datalen=%d, offset=%d, partition=%d", r.URL.String(), costTimeUS, nr, cmData.offset, cmData.partition) w.WriteHeader(http.StatusOK) return }
func (cc *CacheCluster) doDelFromMq(keys []interface{}, delTime, curDelay int64, doneServers map[string]interface{}, firstDel bool) (err error) { /* time to del ? */ now := time.Now().Unix() delta := delTime + curDelay - now logger.Debug("delta=%d, delTime=%d, curDelay=%d, now=%d, firtDel=%t", delta, delTime, curDelay, now, firstDel) if delta > 0 { time.Sleep(time.Duration(delta) * time.Second) now = now + delta } err = cc.cachePool.doDel(keys, doneServers) if err != nil { /* put back to mq */ err = cc.mqClient.addKeys(keys, doneServers, delTime, curDelay, false) if err != nil { return } } else { logger.Notice("del all keys done: cluster=%s, keys=%v, delay=%d", cc.clusterName, keys, curDelay) } if !firstDel { /* not first Del, so all delays has been in mq, we just quit and wait */ return nil } /* ok, it is first del, we add all delays to mq here */ ccc := cc.ccc delay := now - delTime logger.Debug("%d = %d - %d", delay, now, delTime) idx := sort.SearchInts(ccc.delayConfig, int(delay)) if idx == len(ccc.delayConfig) { /* done */ return nil } for ; idx < len(ccc.delayConfig); idx++ { if int64(ccc.delayConfig[idx]) <= delay { continue } // add back to mq with doneServers empty err = cc.mqClient.addKeys(keys, nil, delTime, int64(ccc.delayConfig[idx]), false) if err != nil { return } } return nil }
func (td *TransDi) initTransWorkers() (err error) { for id := uint32(0); id < td.workerNum; id++ { worker := &TransWorker{ moduleName: td.moduleName, workerId: id, backendServers: td.backendServers, protocolConfig: td.protocolConfig, fatalErrorChan: td.fatalErrorChan, inWork: false, ackDataChan: td.ackDataChan, maxRetryTimes: td.maxRetryTimes, failRetryInterval: td.failRetryInterval, } err = worker.init() if err != nil { return } td.transWorkers = append(td.transWorkers, worker) } logger.Notice("all workers have been inited for moudle [%s]", td.moduleName) return nil }
func (httpServer *HttpServer) handleCm(w http.ResponseWriter, r *http.Request) { startTime := time.Now() /* check query */ if r.ContentLength <= 0 { logger.Warning("invalid query, need post data: %s", r.URL.String()) w.WriteHeader(http.StatusBadRequest) return } qv := r.URL.Query() post := make([]byte, r.ContentLength) nr, err := io.ReadFull(r.Body, post) if int64(nr) != r.ContentLength { logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusInternalServerError) return } /* compose CmData */ cmData := httpServer.cdp.fetch() defer httpServer.cdp.put(cmData) cmData.topic = qv.Get("topic") cmData.key = qv.Get("key") cmData.data = post /* commit */ httpServer.cmDataChan <- cmData /* wait res */ <-cmData.cmDoneChan if cmData.err != nil { logger.Warning("fail to commit req: %s, error: %s", r.URL.String(), (*(cmData.err)).Error()) w.WriteHeader(http.StatusInternalServerError) return } endTime := time.Now() costTimeUS := endTime.Sub(startTime) / time.Microsecond // TODO logger.Notice("success process commit: %s, cost_us=%d, datalen=%d, offset=%d, partition=%d", r.URL.String(), costTimeUS, nr, cmData.offset, cmData.partition) w.WriteHeader(http.StatusOK) return }
func (d *Deleter) init() (err error) { err = d.initConfig() if err != nil { return } err = d.initRuntimeConf() if err != nil { return } err = d.initLog() if err != nil { return } err = d.initCacheClusters() if err != nil { return } err = d.initServer() if err != nil { return } logger.Notice("deleter init success") return nil }
func (h *AddHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { startTime := time.Now() /* check query */ if r.ContentLength <= 0 { logger.Warning("invalid query, need post data: %s", r.URL.String()) w.WriteHeader(http.StatusBadRequest) return } // qv := r.URL.Query() bs := h.bs ad := bs.adp.fetch() defer bs.adp.put(ad) ad.buffer.Reset() nr, err := ad.buffer.ReadFrom(r.Body) if int64(nr) != r.ContentLength || err != nil { logger.Warning("fail to read body: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusInternalServerError) return } ad.checksum() ad.Key = "" err = bs.dd.checkDup(ad) id := uint64(0) if err != nil { logger.Warning("fail to dd.checkDup: %s, %s", r.URL.String(), err.Error()) // TODO: continue ? } if len(ad.Key) > 0 { // duplication // done } else { // new data id, err = bs.km.getNewId() if err != nil { logger.Warning("fail to getNewId: %s", err.Error()) w.WriteHeader(http.StatusInternalServerError) return } p, o, err := bs.broker.addNewData(id, ad) if err != nil { logger.Warning("fail to addNewData: %s", err.Error()) w.WriteHeader(http.StatusInternalServerError) return } key, err := bs.km.generateKey(id, p, o) if err != nil { logger.Warning("fail to generateKey: %s", err.Error()) w.WriteHeader(http.StatusInternalServerError) return } ad.Key = key err = bs.dd.insertNew(ad) if err != nil { logger.Warning("fail to dd.insertNew: %s", err.Error()) // only warning here } } // response ok w.WriteHeader(http.StatusOK) _, err = w.Write([]byte(ad.Key)) if err != nil { logger.Warning("fail to write response: %s, %s", r.URL.String(), err.Error()) w.WriteHeader(http.StatusInternalServerError) return } endTime := time.Now() costTimeUS := endTime.Sub(startTime) / time.Microsecond logger.Notice("success process add: %s, cost_us=%d, datalen=%d, id=%d, md5a=%d, md5b=%d, fnv1a32=%d, key=%s", r.URL.String(), costTimeUS, nr, id, ad.md5a, ad.md5b, ad.fnv1a32, ad.Key) return }
func (m *Module) run() { logger.Notice("module [%s] start", m.name) m.transDi.run() // TODO: delete it later m.fatalErrorChan <- errors.New("no error") }
func (d *Deleter) doDel(w http.ResponseWriter, r *http.Request) { /* get topic and method */ logger.Debug("get one req: %s", r.URL.String()) qv := r.URL.Query() topic := qv.Get("topic") if len(topic) == 0 { msg := fmt.Sprintf("invalid query, topic cannot be empty: %s", r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } cc, ok := d.cacheClusters[topic] if !ok { msg := fmt.Sprintf("invalid query, cache cluster is not exist: cluster [%s], %s", topic, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } /* get post data */ if r.ContentLength == 0 { logger.Warning("post data cannot be empty") d.response(w, http.StatusBadRequest, "post data cannot be empty") return } data := make([]byte, r.ContentLength) _, err := io.ReadFull(r.Body, data) if err != nil { msg := fmt.Sprintf("fail to read post data: %s", err.Error()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } /* unpack */ buf := bytes.NewReader(data) buf.Seek(0, 0) msgr := msgp.NewReader(buf) reqi, err := msgr.ReadIntf() if err != nil { msg := fmt.Sprintf("fail to decode post data: %s", err.Error()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } req, ok := reqi.(map[string]interface{}) if !ok { msg := fmt.Sprintf("invalid post data: data need map", r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } keysi, ok := req[gDelKeys] if !ok { msg := fmt.Sprintf("invalid post data: %s not exist: %s", gDelKeys, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } keys, ok := keysi.([]interface{}) if !ok { msg := fmt.Sprintf("invalid post data: %s not array: %s", gDelKeys, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } if len(keys) == 0 { msg := fmt.Sprintf("invalid post data: len of %s array is 0: %s", gDelKeys, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } delTimei, ok := req[gDelDelTime] if !ok { msg := fmt.Sprintf("invalid post data: %s not exist: %s", gDelDelTime, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } delTime, ok := delTimei.(int64) if !ok { delTimet, ok := delTimei.(int) if !ok { delTimetu, ok := delTimei.(uint64) if !ok { msg := fmt.Sprintf("invalid post data: %s not int64 or uint64 or int: %s", gDelDelTime, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } else { delTime = int64(delTimetu) } } else { delTime = int64(delTimet) } } var curDelay int64 curDelayi, ok := req[gDelCurDelay] if !ok { curDelay = 0 } else if curDelay, ok = curDelayi.(int64); !ok { curDelayt, ok := curDelayi.(int) if !ok { curDelaytu, ok := curDelayi.(uint64) if !ok { msg := fmt.Sprintf("invalid post data: %s not int64 or int: %s", gDelCurDelay, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } else { curDelay = int64(curDelaytu) } } else { curDelay = int64(curDelayt) } } var isFromMq bool isFromMqi, ok := req[gDelFromMq] if !ok { isFromMq = false } else if isFromMq, ok = isFromMqi.(bool); !ok { msg := fmt.Sprintf("invalid post data: %s not bool: %s", gDelFromMq, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } var doneServers map[string]interface{} doneServersi, ok := req[gDoneServers] if !ok { doneServers = make(map[string]interface{}) } else if doneServers, ok = doneServersi.(map[string]interface{}); !ok { msg := fmt.Sprintf("invalid post data: %s not map[string]interface{}: %s", gDoneServers, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } var firstDel bool firstDeli, ok := req[gFirstDel] if !ok { firstDel = true } else if firstDel, ok = firstDeli.(bool); !ok { msg := fmt.Sprintf("invalid post data: %s not bool: %s", gFirstDel, r.URL.String()) logger.Warning("%s", msg) d.response(w, http.StatusBadRequest, msg) return } /* do del */ err = cc.doDel(keys, delTime, curDelay, doneServers, isFromMq, firstDel) if err != nil { msg := fmt.Sprintf("fail to del: %s", err.Error()) logger.Warning("%s", msg) d.response(w, http.StatusInternalServerError, msg) return } msg := fmt.Sprintf("success process: %s", r.URL.String()) logger.Notice("%s", msg) d.response(w, http.StatusOK, msg) return }
func (mc *MqClient) addKeys(keys []interface{}, doneServers map[string]interface{}, delTime, nextDelay int64, firstDel bool) (err error) { /* pack data */ data := map[string]interface{}{ gDelKeys: keys, gDelDelTime: delTime, gDelCurDelay: nextDelay, gDelFromMq: true, gFirstDel: firstDel, "method": fmt.Sprintf("%s%d", gDelMethodPrefix, nextDelay), // ktransfer need it } if len(doneServers) > 0 { data[gDoneServers] = doneServers } var buf bytes.Buffer wr := msgp.NewWriter(&buf) err = wr.WriteIntf(data) if err != nil { logger.Warning("fail to msgp.WriteIntf: %s", err.Error()) return } wr.Flush() r := bytes.NewReader(buf.Bytes()) /* write mq */ var mqServerAddrs []string if firstDel && len(mc.idcMqServerAddrs) > 0 { mqServerAddrs = mc.idcMqServerAddrs } else { mqServerAddrs = mc.delMqServerAddrs } toc := make(chan bool, 1) go mc.checkMqTimeout(toc) ns := len(mqServerAddrs) mc.lastServerId = (mc.lastServerId + 1) % ns i := 0 for { r.Seek(0, 0) url := fmt.Sprintf("http://%s?topic=%s&method=%s%d", mqServerAddrs[mc.lastServerId], mc.clusterName, gDelMethodPrefix, nextDelay) req, err1 := http.NewRequest("POST", url, r) if err1 != nil { logger.Warning("fail to http.NewRequest: %s, %s", url, err1.Error()) err = err1 return } rsp, err1 := mc.client.Do(req) if err1 != nil { logger.Warning("fail to http.Client.Do: %s, %s", url, err1.Error()) select { case <-toc: err = errors.New("add back to mq timeout") return default: /* do nothing */ } i++ if i == ns { err = errors.New("fail to add back to mq, all servers have been retried") return } mc.lastServerId = (mc.lastServerId + 1) % ns continue } else { rsp.Body.Close() logger.Notice("success add back to mq: %s", url) return nil } } }