Beispiel #1
0
func (s *Server) groupUpdate(group string, queue string,
	write string, read string, url string, ips string) string {

	config, err := s.queue.GetSingleGroup(group, queue)
	if err != nil {
		log.Debugf("GetSingleGroup err:%s", errors.ErrorStack(err))
		return `{"action":"update","result":false}`
	}
	if write != "" {
		w, err := strconv.ParseBool(write)
		if err == nil {
			config.Write = w
		}
	}
	if read != "" {
		r, err := strconv.ParseBool(read)
		if err == nil {
			config.Read = r
		}
	}
	if url != "" {
		config.Url = url
	}
	if ips != "" {
		config.Ips = strings.Split(ips, ",")
	}

	err = s.queue.UpdateGroup(group, queue, config.Write, config.Read, config.Url, config.Ips)
	if err != nil {
		log.Debugf("groupUpdate failed: %s", errors.ErrorStack(err))
		return `{"action":"update","result":false}`
	}
	return `{"action":"update","result":true}`
}
Beispiel #2
0
func (s *Server) monitorHandler(w http.ResponseWriter, r *http.Request) {

	r.ParseForm()
	monitorType := r.FormValue("type")
	queue := r.FormValue("queue")
	group := r.FormValue("group")

	end := time.Now().Unix()
	start := end - 5*60 //5min
	interval := int64(1)

	startTime := r.FormValue("start")
	if startTime != "" {
		start, _ = strconv.ParseInt(startTime, 10, 0)
	}
	endTime := r.FormValue("end")
	if endTime != "" {
		end, _ = strconv.ParseInt(endTime, 10, 0)
	}
	intervalTime := r.FormValue("interval")
	if intervalTime != "" {
		interval, _ = strconv.ParseInt(intervalTime, 10, 0)
	}

	var result string

	switch monitorType {
	case "send":
		m, err := s.queue.GetSendMetrics(queue, group, start, end, interval)
		if err != nil {
			log.Debug("GetSendMetrics err: %s", errors.ErrorStack(err))
			return
		}
		sm, err := json.Marshal(m)
		if err != nil {
			log.Debugf("GetSendMetrics Marshal err: %s", err)
			return
		}
		result = string(sm)
	case "receive":
		m, err := s.queue.GetReceiveMetrics(queue, group, start, end, interval)
		if err != nil {
			log.Debug("GetReceiveMetrics err: %s", errors.ErrorStack(err))
			return
		}
		rm, err := json.Marshal(m)
		if err != nil {
			log.Debugf("GetReceiveMetrics Marshal err: %s", err)
			return
		}
		result = string(rm)
	default:
		result = "error, param type=" + monitorType + " not support!"
	}
	fmt.Fprintf(w, result)
}
Beispiel #3
0
func (s *Server) queueLookup(queue string, biz string) string {
	r, err := s.queue.Lookup(queue, biz)
	if err != nil {
		log.Debugf("LookupQueue err:%s", errors.ErrorStack(err))
		return "[]"
	}
	result, err := json.Marshal(r)
	if err != nil {
		log.Debugf("queueLookup Marshal err:%s", err)
		return "[]"
	}
	return string(result)
}
Beispiel #4
0
func (s *Server) groupLookup(group string) string {
	r, err := s.queue.LookupGroup(group)
	if err != nil {
		log.Debugf("LookupGroup err: %s", errors.ErrorStack(err))
		return "[]"
	}
	result, err := json.Marshal(r)
	if err != nil {
		log.Debugf("LookupGroup Marshal err: %s", err)
		return "[]"
	}
	return string(result)
}
Beispiel #5
0
//Delete a queue by name
func (m *Metadata) DelQueue(queue string) error {

	mutex := zk.NewLock(m.zkClient.Conn, m.operationPath, zk.WorldACL(zk.PermAll))
	if err := mutex.Lock(); err != nil {
		return errors.Trace(err)
	}
	defer mutex.Unlock()

	if err := m.RefreshMetadata(); err != nil {
		return errors.Trace(err)
	}

	can, err := m.canDeleteQueue(queue)
	if err != nil {
		return errors.Trace(err)
	}
	if !can {
		return errors.NotValidf("DeleteQueue queue:%s has one or more group", queue)
	}

	path := m.buildQueuePath(queue)
	log.Debugf("del queue, zk path:%s", path)
	if err := m.zkClient.DeleteRec(path); err != nil {
		return errors.Trace(err)
	}
	delete(m.queueConfigs, queue)
	if err := m.manager.DeleteTopic(queue); err != nil {
		return errors.Trace(err)
	}
	return nil
}
Beispiel #6
0
//Add a queue by name
func (m *Metadata) AddQueue(queue string) error {

	mutex := zk.NewLock(m.zkClient.Conn, m.operationPath, zk.WorldACL(zk.PermAll))
	if err := mutex.Lock(); err != nil {
		return errors.Trace(err)
	}
	defer mutex.Unlock()

	if err := m.RefreshMetadata(); err != nil {
		return errors.Trace(err)
	}

	if exist := m.ExistQueue(queue); exist {
		return errors.AlreadyExistsf("CreateQueue queue:%s ", queue)
	}

	if err := m.manager.CreateTopic(queue, int32(m.config.KafkaReplications),
		int32(m.config.KafkaPartitions)); err != nil {
		return errors.Trace(err)
	}

	path := m.buildQueuePath(queue)
	data := ""
	log.Debugf("add queue, zk path:%s, data:%s", path, data)

	if err := m.zkClient.CreateRec(path, data, 0); err != nil {
		return errors.Trace(err)
	}
	return nil
}
Beispiel #7
0
func (m *Metadata) UpdateGroupConfig(group string, queue string,
	write bool, read bool, url string, ips []string) error {

	mutex := zk.NewLock(m.zkClient.Conn, m.operationPath, zk.WorldACL(zk.PermAll))
	if err := mutex.Lock(); err != nil {
		return errors.Trace(err)
	}
	defer mutex.Unlock()

	if err := m.RefreshMetadata(); err != nil {
		return errors.Trace(err)
	}

	if exist := m.ExistGroup(queue, group); !exist {
		return errors.NotFoundf("queue : %q, group: %q", queue, group)
	}

	path := m.buildConfigPath(group, queue)
	groupConfig := GroupConfig{
		Group: group,
		Queue: queue,
		Write: write,
		Read:  read,
		Url:   url,
		Ips:   ips,
	}
	data := groupConfig.String()
	log.Debugf("update group config, zk path:%s, data:%s", path, data)
	if err := m.zkClient.Set(path, data); err != nil {
		return errors.Trace(err)
	}
	return nil
}
Beispiel #8
0
func (s *Server) groupRemove(group string, queue string) string {
	err := s.queue.DeleteGroup(group, queue)
	if err != nil {
		log.Debugf("groupRemove failed: %s", errors.ErrorStack(err))
		return `{"action":"remove","result":false}`
	}
	return `{"action":"remove","result":true}`
}
Beispiel #9
0
func (s *Server) queueUpdate(queue string) string {
	err := s.queue.Update(queue)
	if err != nil {
		log.Debugf("UpdateQueue err:%s", errors.ErrorStack(err))
		return `{"action":"update","result":false}`
	}
	return `{"action":"update","result":true}`
}
Beispiel #10
0
func (ms *McServer) Start() error {

	var err error
	ms.listener, err = utils.Listen("tcp", fmt.Sprintf(":%s", ms.port))
	if err != nil {
		return errors.Trace(err)
	}
	log.Debugf("memcached protocol server start on %s port.", ms.port)
	go ms.mainLoop()
	return nil
}
Beispiel #11
0
func (s *Server) msgSend(queue string, group string, msg string) string {
	var result string
	_, err := s.queue.SendMessage(queue, group, []byte(msg), 0)
	if err != nil {
		log.Debugf("msgSend failed: %s", errors.ErrorStack(err))
		result = err.Error()
	} else {
		result = `{"action":"send","result":true}`
	}
	return result
}
Beispiel #12
0
func (ms *McServer) Stop() {
	if !atomic.CompareAndSwapInt32(&ms.stopping, 0, 1) {
		return
	}
	//Make on processing commamd to run over
	time.Sleep(200 * time.Millisecond)
	if err := ms.listener.Close(); err != nil {
		log.Errorf("mc server listener close failed:%s", err)
		return
	}
	ms.DrainConn()
	for ms.listener.GetRemain() != 0 {
		time.Sleep(time.Millisecond)
	}
	log.Debugf("mc protocol server stop.")
}
Beispiel #13
0
func (s *Server) groupAdd(group string, queue string, write string, read string, url string, ips string) string {

	w, _ := strconv.ParseBool(write)
	r, _ := strconv.ParseBool(read)
	ips_array := strings.Split(ips, ",")

	if url == "" {
		url = fmt.Sprintf("%s.%s.intra.weibo.com", group, queue)
	}

	err := s.queue.AddGroup(group, queue, w, r, url, ips_array)
	if err != nil {
		log.Debugf("AddGroup failed: %s", errors.ErrorStack(err))
		return `{"action":"add","result":false}`
	}
	return `{"action":"add","result":true}`
}
Beispiel #14
0
func (s *Server) msgReceive(queue string, group string) string {
	var result string
	id, data, _, err := s.queue.RecvMessage(queue, group)
	if err != nil {
		log.Debugf("msgReceive failed: %s", errors.ErrorStack(err))
		result = err.Error()
	} else {
		err = s.queue.AckMessage(queue, group, id)
		if err != nil {
			log.Warnf("ack message queue:%q group:%q id:%q err:%s", queue, group, id, err)
			result = err.Error()
		} else {
			result = `{"action":"receive","msg":"` + string(data) + `"}`
		}
	}
	return result
}
Beispiel #15
0
func (q *queueImp) RecvMessage(queue string, group string) (string, []byte, uint64, error) {
	var err error
	start := time.Now()
	exist := q.metadata.ExistGroup(queue, group)
	if !exist {
		return "", nil, 0, errors.NotFoundf("queue : %q , group: %q", queue, group)
	}

	owner := fmt.Sprintf("%s@%s", queue, group)
	q.mu.Lock()
	consumer, ok := q.consumerMap[owner]
	if !ok {
		consumer, err = kafka.NewConsumer(q.metadata.manager.BrokerAddrs(), q.clusterConfig, queue, group)
		if err != nil {
			q.mu.Unlock()
			return "", nil, 0, errors.Trace(err)
		}
		q.consumerMap[owner] = consumer
	}
	q.mu.Unlock()

	msg, err := consumer.Recv()
	if err != nil {
		return "", nil, 0, errors.Trace(err)
	}

	var sequenceID, flag uint64
	tokens := strings.Split(string(msg.Key), ":")
	sequenceID, _ = strconv.ParseUint(tokens[0], 16, 64)
	if len(tokens) > 1 {
		flag, _ = strconv.ParseUint(tokens[1], 16, 32)
	}

	messageID := fmt.Sprintf("%x:%s:%s:%x:%x", sequenceID, queue, group, msg.Partition, msg.Offset)

	end := time.Now()
	cost := end.Sub(start).Nanoseconds() / 1e6
	delay := end.UnixNano()/1e6 - baseTime - int64((sequenceID>>24)&0xFFFFFFFFFF)

	metrics.Add(queue+"#"+group+"#recv", 1, cost, delay)

	log.Debugf("recv %s:%s key %s id %s cost %d delay %d", queue, group, string(msg.Key), messageID, cost, delay)
	return messageID, msg.Value, flag, nil
}
Beispiel #16
0
func (ms *McServer) mainLoop() {

	for atomic.LoadInt32(&ms.stopping) == 0 {
		conn, err := ms.listener.Accept()
		if err != nil {
			log.Errorf("mc server accept error: %s", err)
			continue
		}
		if atomic.LoadInt32(&ms.stopping) != 0 {
			conn.Close()
			return
		}
		log.Debugf("mc server new client: %s", conn.RemoteAddr())
		ms.mu.Lock()
		ms.connPool[conn] = conn
		ms.mu.Unlock()
		go ms.connLoop(conn)
	}
}
Beispiel #17
0
func (ms *McServer) connLoop(conn net.Conn) {
	defer func(conn net.Conn) {
		log.Debugf("mc client closed :%s", conn.RemoteAddr())
		ms.mu.Lock()
		delete(ms.connPool, conn)
		ms.mu.Unlock()
		conn.Close()
		if err := recover(); err != nil {
			log.Errorf("mc connLoop panic error: %s", err)
		}
	}(conn)

	br := bufio.NewReaderSize(conn, ms.recvBuffSize)
	bw := bufio.NewWriterSize(conn, ms.sendBuffSize)

	for atomic.LoadInt32(&ms.stopping) == 0 {
		data, err := br.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				return
			}
			log.Warnf("mc server ReadLine err:%s", err)
			return
		}

		tokens := strings.Split(strings.TrimSpace(data), " ")
		cmd := tokens[0]
		command, exists := commands[cmd]
		if !exists {
			command = commandUnkown
			cmd = "unsupported"
		}
		metrics.Add(cmd, 1)
		err = command(ms.queue, tokens, br, bw)
		bw.Flush()
		if err != nil {
			//command返回错误一定是不能容忍的错误,需要退出循环关闭连接,防止将后续有效数据的格式都破坏掉
			log.Errorf("mc bad command:%s", errors.ErrorStack(err))
			return
		}
	}
}
Beispiel #18
0
func (q *queueImp) AckMessage(queue string, group string, id string) error {
	start := time.Now()

	if exist := q.metadata.ExistGroup(queue, group); !exist {
		return errors.NotFoundf("queue : %q , group: %q", queue, group)
	}

	owner := fmt.Sprintf("%s@%s", queue, group)
	q.mu.Lock()
	consumer, ok := q.consumerMap[owner]
	q.mu.Unlock()
	if !ok {
		return errors.NotFoundf("group consumer")
	}

	tokens := strings.Split(id, ":")
	if len(tokens) != 5 {
		return errors.NotValidf("message ID : %q", id)
	}

	partition, err := strconv.ParseInt(tokens[3], 16, 32)
	if err != nil {
		return errors.NotValidf("message ID : %q", id)
	}
	offset, err := strconv.ParseInt(tokens[4], 16, 64)
	if err != nil {
		return errors.NotValidf("message ID : %q", id)
	}

	err = consumer.Ack(int32(partition), offset)
	if err != nil {
		return errors.Trace(err)
	}

	cost := time.Now().Sub(start).Nanoseconds() / 1e6
	log.Debugf("ack %s:%s key nil id %s cost %d", queue, group, id, cost)
	return nil
}
Beispiel #19
0
func (q *queueImp) SendMessage(queue string, group string, data []byte, flag uint64) (string, error) {
	start := time.Now()

	exist := q.metadata.ExistGroup(queue, group)
	if !exist {
		return "", errors.NotFoundf("queue : %q , group: %q", queue, group)
	}

	sequenceID := q.idGenerator.Get()
	key := fmt.Sprintf("%x:%x", sequenceID, flag)

	partition, offset, err := q.producer.Send(queue, []byte(key), data)
	if err != nil {
		return "", errors.Trace(err)
	}
	messageID := fmt.Sprintf("%x:%s:%s:%x:%x", sequenceID, queue, group, partition, offset)
	cost := time.Now().Sub(start).Nanoseconds() / 1e6

	metrics.Add(queue+"#"+group+"#sent", 1, cost)

	log.Debugf("send %s:%s key %s id %s cost %d", queue, group, key, messageID, cost)
	return messageID, nil
}
Beispiel #20
0
func (m *Metadata) DeleteGroupConfig(group string, queue string) error {

	mutex := zk.NewLock(m.zkClient.Conn, m.operationPath, zk.WorldACL(zk.PermAll))
	if err := mutex.Lock(); err != nil {
		return errors.Trace(err)
	}
	defer mutex.Unlock()

	if err := m.RefreshMetadata(); err != nil {
		return errors.Trace(err)
	}

	if exist := m.ExistGroup(queue, group); !exist {
		return errors.NotFoundf("queue : %q, group : %q", queue, group)
	}

	path := m.buildConfigPath(group, queue)
	log.Debugf("delete group config, zk path:%s", path)
	if err := m.zkClient.DeleteRec(path); err != nil {
		return errors.Trace(err)
	}
	return nil
}