示例#1
0
func (q *queueImp) RecvMessage(queue string, group string) (string, []byte, uint64, error) {
	var err error
	start := time.Now()
	exist := q.metadata.ExistGroup(queue, group)
	if !exist {
		return "", nil, 0, errors.NotFoundf("queue : %q , group: %q", queue, group)
	}

	owner := fmt.Sprintf("%s@%s", queue, group)
	q.mu.Lock()
	consumer, ok := q.consumerMap[owner]
	if !ok {
		consumer, err = kafka.NewConsumer(q.metadata.manager.BrokerAddrs(), q.clusterConfig, queue, group)
		if err != nil {
			q.mu.Unlock()
			return "", nil, 0, errors.Trace(err)
		}
		q.consumerMap[owner] = consumer
	}
	q.mu.Unlock()

	msg, err := consumer.Recv()
	if err != nil {
		return "", nil, 0, errors.Trace(err)
	}

	var sequenceID, flag uint64
	tokens := strings.Split(string(msg.Key), ":")
	sequenceID, _ = strconv.ParseUint(tokens[0], 16, 64)
	if len(tokens) > 1 {
		flag, _ = strconv.ParseUint(tokens[1], 16, 32)
	}

	messageID := fmt.Sprintf("%x:%s:%s:%x:%x", sequenceID, queue, group, msg.Partition, msg.Offset)

	end := time.Now()
	cost := end.Sub(start).Nanoseconds() / 1e6
	delay := end.UnixNano()/1e6 - baseTime - int64((sequenceID>>24)&0xFFFFFFFFFF)

	metrics.Add(queue+"#"+group+"#recv", 1, cost, delay)

	log.Debugf("recv %s:%s key %s id %s cost %d delay %d", queue, group, string(msg.Key), messageID, cost, delay)
	return messageID, msg.Value, flag, nil
}
示例#2
0
func (ms *McServer) connLoop(conn net.Conn) {
	defer func(conn net.Conn) {
		log.Debugf("mc client closed :%s", conn.RemoteAddr())
		ms.mu.Lock()
		delete(ms.connPool, conn)
		ms.mu.Unlock()
		conn.Close()
		if err := recover(); err != nil {
			log.Errorf("mc connLoop panic error: %s", err)
		}
	}(conn)

	br := bufio.NewReaderSize(conn, ms.recvBuffSize)
	bw := bufio.NewWriterSize(conn, ms.sendBuffSize)

	for atomic.LoadInt32(&ms.stopping) == 0 {
		data, err := br.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				return
			}
			log.Warnf("mc server ReadLine err:%s", err)
			return
		}

		tokens := strings.Split(strings.TrimSpace(data), " ")
		cmd := tokens[0]
		command, exists := commands[cmd]
		if !exists {
			command = commandUnkown
			cmd = "unsupported"
		}
		metrics.Add(cmd, 1)
		err = command(ms.queue, tokens, br, bw)
		bw.Flush()
		if err != nil {
			//command返回错误一定是不能容忍的错误,需要退出循环关闭连接,防止将后续有效数据的格式都破坏掉
			log.Errorf("mc bad command:%s", errors.ErrorStack(err))
			return
		}
	}
}
示例#3
0
func (q *queueImp) SendMessage(queue string, group string, data []byte, flag uint64) (string, error) {
	start := time.Now()

	exist := q.metadata.ExistGroup(queue, group)
	if !exist {
		return "", errors.NotFoundf("queue : %q , group: %q", queue, group)
	}

	sequenceID := q.idGenerator.Get()
	key := fmt.Sprintf("%x:%x", sequenceID, flag)

	partition, offset, err := q.producer.Send(queue, []byte(key), data)
	if err != nil {
		return "", errors.Trace(err)
	}
	messageID := fmt.Sprintf("%x:%s:%s:%x:%x", sequenceID, queue, group, partition, offset)
	cost := time.Now().Sub(start).Nanoseconds() / 1e6

	metrics.Add(queue+"#"+group+"#sent", 1, cost)

	log.Debugf("send %s:%s key %s id %s cost %d", queue, group, key, messageID, cost)
	return messageID, nil
}