func (s *Server) groupUpdate(group string, queue string,
write string, read string, url string, ips string) string {
config, err := s.queue.GetSingleGroup(group, queue)
if err != nil {
log.Debugf("GetSingleGroup err:%s", errors.ErrorStack(err))
return `{"action":"update","result":false}`
}
if write != "" {
w, err := strconv.ParseBool(write)
if err == nil {
config.Write = w
}
}
if read != "" {
r, err := strconv.ParseBool(read)
if err == nil {
config.Read = r
}
}
if url != "" {
config.Url = url
}
if ips != "" {
config.Ips = strings.Split(ips, ",")
}
err = s.queue.UpdateGroup(group, queue, config.Write, config.Read, config.Url, config.Ips)
if err != nil {
log.Debugf("groupUpdate failed: %s", errors.ErrorStack(err))
return `{"action":"update","result":false}`
}
return `{"action":"update","result":true}`
}
func (s *Server) monitorHandler(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
monitorType := r.FormValue("type")
queue := r.FormValue("queue")
group := r.FormValue("group")
end := time.Now().Unix()
start := end - 5*60 //5min
interval := int64(1)
startTime := r.FormValue("start")
if startTime != "" {
start, _ = strconv.ParseInt(startTime, 10, 0)
}
endTime := r.FormValue("end")
if endTime != "" {
end, _ = strconv.ParseInt(endTime, 10, 0)
}
intervalTime := r.FormValue("interval")
if intervalTime != "" {
interval, _ = strconv.ParseInt(intervalTime, 10, 0)
}
var result string
switch monitorType {
case "send":
m, err := s.queue.GetSendMetrics(queue, group, start, end, interval)
if err != nil {
log.Debug("GetSendMetrics err: %s", errors.ErrorStack(err))
return
}
sm, err := json.Marshal(m)
if err != nil {
log.Debugf("GetSendMetrics Marshal err: %s", err)
return
}
result = string(sm)
case "receive":
m, err := s.queue.GetReceiveMetrics(queue, group, start, end, interval)
if err != nil {
log.Debug("GetReceiveMetrics err: %s", errors.ErrorStack(err))
return
}
rm, err := json.Marshal(m)
if err != nil {
log.Debugf("GetReceiveMetrics Marshal err: %s", err)
return
}
result = string(rm)
default:
result = "error, param type=" + monitorType + " not support!"
}
fmt.Fprintf(w, result)
}
func (s *Server) queueLookup(queue string, biz string) string {
r, err := s.queue.Lookup(queue, biz)
if err != nil {
log.Debugf("LookupQueue err:%s", errors.ErrorStack(err))
return "[]"
}
result, err := json.Marshal(r)
if err != nil {
log.Debugf("queueLookup Marshal err:%s", err)
return "[]"
}
return string(result)
}
func (s *Server) groupLookup(group string) string {
r, err := s.queue.LookupGroup(group)
if err != nil {
log.Debugf("LookupGroup err: %s", errors.ErrorStack(err))
return "[]"
}
result, err := json.Marshal(r)
if err != nil {
log.Debugf("LookupGroup Marshal err: %s", err)
return "[]"
}
return string(result)
}
//Delete a queue by name
func (m *Metadata) DelQueue(queue string) error {
mutex := zk.NewLock(m.zkClient.Conn, m.operationPath, zk.WorldACL(zk.PermAll))
if err := mutex.Lock(); err != nil {
return errors.Trace(err)
}
defer mutex.Unlock()
if err := m.RefreshMetadata(); err != nil {
return errors.Trace(err)
}
can, err := m.canDeleteQueue(queue)
if err != nil {
return errors.Trace(err)
}
if !can {
return errors.NotValidf("DeleteQueue queue:%s has one or more group", queue)
}
path := m.buildQueuePath(queue)
log.Debugf("del queue, zk path:%s", path)
if err := m.zkClient.DeleteRec(path); err != nil {
return errors.Trace(err)
}
delete(m.queueConfigs, queue)
if err := m.manager.DeleteTopic(queue); err != nil {
return errors.Trace(err)
}
return nil
}
//Add a queue by name
func (m *Metadata) AddQueue(queue string) error {
mutex := zk.NewLock(m.zkClient.Conn, m.operationPath, zk.WorldACL(zk.PermAll))
if err := mutex.Lock(); err != nil {
return errors.Trace(err)
}
defer mutex.Unlock()
if err := m.RefreshMetadata(); err != nil {
return errors.Trace(err)
}
if exist := m.ExistQueue(queue); exist {
return errors.AlreadyExistsf("CreateQueue queue:%s ", queue)
}
if err := m.manager.CreateTopic(queue, int32(m.config.KafkaReplications),
int32(m.config.KafkaPartitions)); err != nil {
return errors.Trace(err)
}
path := m.buildQueuePath(queue)
data := ""
log.Debugf("add queue, zk path:%s, data:%s", path, data)
if err := m.zkClient.CreateRec(path, data, 0); err != nil {
return errors.Trace(err)
}
return nil
}
func (m *Metadata) UpdateGroupConfig(group string, queue string,
write bool, read bool, url string, ips []string) error {
mutex := zk.NewLock(m.zkClient.Conn, m.operationPath, zk.WorldACL(zk.PermAll))
if err := mutex.Lock(); err != nil {
return errors.Trace(err)
}
defer mutex.Unlock()
if err := m.RefreshMetadata(); err != nil {
return errors.Trace(err)
}
if exist := m.ExistGroup(queue, group); !exist {
return errors.NotFoundf("queue : %q, group: %q", queue, group)
}
path := m.buildConfigPath(group, queue)
groupConfig := GroupConfig{
Group: group,
Queue: queue,
Write: write,
Read: read,
Url: url,
Ips: ips,
}
data := groupConfig.String()
log.Debugf("update group config, zk path:%s, data:%s", path, data)
if err := m.zkClient.Set(path, data); err != nil {
return errors.Trace(err)
}
return nil
}
func (s *Server) groupRemove(group string, queue string) string {
err := s.queue.DeleteGroup(group, queue)
if err != nil {
log.Debugf("groupRemove failed: %s", errors.ErrorStack(err))
return `{"action":"remove","result":false}`
}
return `{"action":"remove","result":true}`
}
func (s *Server) queueUpdate(queue string) string {
err := s.queue.Update(queue)
if err != nil {
log.Debugf("UpdateQueue err:%s", errors.ErrorStack(err))
return `{"action":"update","result":false}`
}
return `{"action":"update","result":true}`
}
func (ms *McServer) Start() error {
var err error
ms.listener, err = utils.Listen("tcp", fmt.Sprintf(":%s", ms.port))
if err != nil {
return errors.Trace(err)
}
log.Debugf("memcached protocol server start on %s port.", ms.port)
go ms.mainLoop()
return nil
}
func (s *Server) msgSend(queue string, group string, msg string) string {
var result string
_, err := s.queue.SendMessage(queue, group, []byte(msg), 0)
if err != nil {
log.Debugf("msgSend failed: %s", errors.ErrorStack(err))
result = err.Error()
} else {
result = `{"action":"send","result":true}`
}
return result
}
func (ms *McServer) Stop() {
if !atomic.CompareAndSwapInt32(&ms.stopping, 0, 1) {
return
}
//Make on processing commamd to run over
time.Sleep(200 * time.Millisecond)
if err := ms.listener.Close(); err != nil {
log.Errorf("mc server listener close failed:%s", err)
return
}
ms.DrainConn()
for ms.listener.GetRemain() != 0 {
time.Sleep(time.Millisecond)
}
log.Debugf("mc protocol server stop.")
}
func (s *Server) groupAdd(group string, queue string, write string, read string, url string, ips string) string {
w, _ := strconv.ParseBool(write)
r, _ := strconv.ParseBool(read)
ips_array := strings.Split(ips, ",")
if url == "" {
url = fmt.Sprintf("%s.%s.intra.weibo.com", group, queue)
}
err := s.queue.AddGroup(group, queue, w, r, url, ips_array)
if err != nil {
log.Debugf("AddGroup failed: %s", errors.ErrorStack(err))
return `{"action":"add","result":false}`
}
return `{"action":"add","result":true}`
}
func (s *Server) msgReceive(queue string, group string) string {
var result string
id, data, _, err := s.queue.RecvMessage(queue, group)
if err != nil {
log.Debugf("msgReceive failed: %s", errors.ErrorStack(err))
result = err.Error()
} else {
err = s.queue.AckMessage(queue, group, id)
if err != nil {
log.Warnf("ack message queue:%q group:%q id:%q err:%s", queue, group, id, err)
result = err.Error()
} else {
result = `{"action":"receive","msg":"` + string(data) + `"}`
}
}
return result
}
func (q *queueImp) RecvMessage(queue string, group string) (string, []byte, uint64, error) {
var err error
start := time.Now()
exist := q.metadata.ExistGroup(queue, group)
if !exist {
return "", nil, 0, errors.NotFoundf("queue : %q , group: %q", queue, group)
}
owner := fmt.Sprintf("%s@%s", queue, group)
q.mu.Lock()
consumer, ok := q.consumerMap[owner]
if !ok {
consumer, err = kafka.NewConsumer(q.metadata.manager.BrokerAddrs(), q.clusterConfig, queue, group)
if err != nil {
q.mu.Unlock()
return "", nil, 0, errors.Trace(err)
}
q.consumerMap[owner] = consumer
}
q.mu.Unlock()
msg, err := consumer.Recv()
if err != nil {
return "", nil, 0, errors.Trace(err)
}
var sequenceID, flag uint64
tokens := strings.Split(string(msg.Key), ":")
sequenceID, _ = strconv.ParseUint(tokens[0], 16, 64)
if len(tokens) > 1 {
flag, _ = strconv.ParseUint(tokens[1], 16, 32)
}
messageID := fmt.Sprintf("%x:%s:%s:%x:%x", sequenceID, queue, group, msg.Partition, msg.Offset)
end := time.Now()
cost := end.Sub(start).Nanoseconds() / 1e6
delay := end.UnixNano()/1e6 - baseTime - int64((sequenceID>>24)&0xFFFFFFFFFF)
metrics.Add(queue+"#"+group+"#recv", 1, cost, delay)
log.Debugf("recv %s:%s key %s id %s cost %d delay %d", queue, group, string(msg.Key), messageID, cost, delay)
return messageID, msg.Value, flag, nil
}
func (ms *McServer) mainLoop() {
for atomic.LoadInt32(&ms.stopping) == 0 {
conn, err := ms.listener.Accept()
if err != nil {
log.Errorf("mc server accept error: %s", err)
continue
}
if atomic.LoadInt32(&ms.stopping) != 0 {
conn.Close()
return
}
log.Debugf("mc server new client: %s", conn.RemoteAddr())
ms.mu.Lock()
ms.connPool[conn] = conn
ms.mu.Unlock()
go ms.connLoop(conn)
}
}
func (ms *McServer) connLoop(conn net.Conn) {
defer func(conn net.Conn) {
log.Debugf("mc client closed :%s", conn.RemoteAddr())
ms.mu.Lock()
delete(ms.connPool, conn)
ms.mu.Unlock()
conn.Close()
if err := recover(); err != nil {
log.Errorf("mc connLoop panic error: %s", err)
}
}(conn)
br := bufio.NewReaderSize(conn, ms.recvBuffSize)
bw := bufio.NewWriterSize(conn, ms.sendBuffSize)
for atomic.LoadInt32(&ms.stopping) == 0 {
data, err := br.ReadString('\n')
if err != nil {
if err == io.EOF {
return
}
log.Warnf("mc server ReadLine err:%s", err)
return
}
tokens := strings.Split(strings.TrimSpace(data), " ")
cmd := tokens[0]
command, exists := commands[cmd]
if !exists {
command = commandUnkown
cmd = "unsupported"
}
metrics.Add(cmd, 1)
err = command(ms.queue, tokens, br, bw)
bw.Flush()
if err != nil {
//command返回错误一定是不能容忍的错误，需要退出循环关闭连接，防止将后续有效数据的格式都破坏掉
log.Errorf("mc bad command:%s", errors.ErrorStack(err))
return
}
}
}
func (q *queueImp) AckMessage(queue string, group string, id string) error {
start := time.Now()
if exist := q.metadata.ExistGroup(queue, group); !exist {
return errors.NotFoundf("queue : %q , group: %q", queue, group)
}
owner := fmt.Sprintf("%s@%s", queue, group)
q.mu.Lock()
consumer, ok := q.consumerMap[owner]
q.mu.Unlock()
if !ok {
return errors.NotFoundf("group consumer")
}
tokens := strings.Split(id, ":")
if len(tokens) != 5 {
return errors.NotValidf("message ID : %q", id)
}
partition, err := strconv.ParseInt(tokens[3], 16, 32)
if err != nil {
return errors.NotValidf("message ID : %q", id)
}
offset, err := strconv.ParseInt(tokens[4], 16, 64)
if err != nil {
return errors.NotValidf("message ID : %q", id)
}
err = consumer.Ack(int32(partition), offset)
if err != nil {
return errors.Trace(err)
}
cost := time.Now().Sub(start).Nanoseconds() / 1e6
log.Debugf("ack %s:%s key nil id %s cost %d", queue, group, id, cost)
return nil
}
func (q *queueImp) SendMessage(queue string, group string, data []byte, flag uint64) (string, error) {
start := time.Now()
exist := q.metadata.ExistGroup(queue, group)
if !exist {
return "", errors.NotFoundf("queue : %q , group: %q", queue, group)
}
sequenceID := q.idGenerator.Get()
key := fmt.Sprintf("%x:%x", sequenceID, flag)
partition, offset, err := q.producer.Send(queue, []byte(key), data)
if err != nil {
return "", errors.Trace(err)
}
messageID := fmt.Sprintf("%x:%s:%s:%x:%x", sequenceID, queue, group, partition, offset)
cost := time.Now().Sub(start).Nanoseconds() / 1e6
metrics.Add(queue+"#"+group+"#sent", 1, cost)
log.Debugf("send %s:%s key %s id %s cost %d", queue, group, key, messageID, cost)
return messageID, nil
}
func (m *Metadata) DeleteGroupConfig(group string, queue string) error {
mutex := zk.NewLock(m.zkClient.Conn, m.operationPath, zk.WorldACL(zk.PermAll))
if err := mutex.Lock(); err != nil {
return errors.Trace(err)
}
defer mutex.Unlock()
if err := m.RefreshMetadata(); err != nil {
return errors.Trace(err)
}
if exist := m.ExistGroup(queue, group); !exist {
return errors.NotFoundf("queue : %q, group : %q", queue, group)
}
path := m.buildConfigPath(group, queue)
log.Debugf("delete group config, zk path:%s", path)
if err := m.zkClient.DeleteRec(path); err != nil {
return errors.Trace(err)
}
return nil
}