Example #1
0
func (ms *McServer) connLoop(conn net.Conn) {
	defer func(conn net.Conn) {
		log.Debugf("mc client closed :%s", conn.RemoteAddr())
		ms.mu.Lock()
		delete(ms.connPool, conn)
		ms.mu.Unlock()
		conn.Close()
		if err := recover(); err != nil {
			log.Errorf("mc connLoop panic error: %s", err)
		}
	}(conn)

	br := bufio.NewReaderSize(conn, ms.recvBuffSize)
	bw := bufio.NewWriterSize(conn, ms.sendBuffSize)

	for atomic.LoadInt32(&ms.stopping) == 0 {
		data, err := br.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				return
			}
			log.Warnf("mc server ReadLine err:%s", err)
			return
		}

		tokens := strings.Split(strings.TrimSpace(data), " ")
		cmd := tokens[0]
		command, exists := commands[cmd]
		if !exists {
			command = commandUnkown
			cmd = "unsupported"
		}
		metrics.Add(cmd, 1)
		err = command(ms.queue, tokens, br, bw)
		bw.Flush()
		if err != nil {
			//command返回错误一定是不能容忍的错误,需要退出循环关闭连接,防止将后续有效数据的格式都破坏掉
			log.Errorf("mc bad command:%s", errors.ErrorStack(err))
			return
		}
	}
}
Example #2
0
func (q *queueImp) Close() {
	q.mu.Lock()
	defer q.mu.Unlock()

	err := q.producer.Close()
	if err != nil {
		log.Errorf("close producer err: %s", err)
	}

	for name, consumer := range q.consumerMap {
		err = consumer.Close()
		if err != nil {
			log.Errorf("close consumer %s err: %s", name, err)
		}
		delete(q.consumerMap, name)
	}

	err = q.monitor.Close()
	if err != nil {
		log.Errorf("close monitor err: %s", err)
	}

	q.metadata.Close()
}
Example #3
0
func (ms *McServer) Stop() {
	if !atomic.CompareAndSwapInt32(&ms.stopping, 0, 1) {
		return
	}
	//Make on processing commamd to run over
	time.Sleep(200 * time.Millisecond)
	if err := ms.listener.Close(); err != nil {
		log.Errorf("mc server listener close failed:%s", err)
		return
	}
	ms.DrainConn()
	for ms.listener.GetRemain() != 0 {
		time.Sleep(time.Millisecond)
	}
	log.Debugf("mc protocol server stop.")
}
Example #4
0
func (ms *McServer) mainLoop() {

	for atomic.LoadInt32(&ms.stopping) == 0 {
		conn, err := ms.listener.Accept()
		if err != nil {
			log.Errorf("mc server accept error: %s", err)
			continue
		}
		if atomic.LoadInt32(&ms.stopping) != 0 {
			conn.Close()
			return
		}
		log.Debugf("mc server new client: %s", conn.RemoteAddr())
		ms.mu.Lock()
		ms.connPool[conn] = conn
		ms.mu.Unlock()
		go ms.connLoop(conn)
	}
}
Example #5
0
func NewConsumer(brokerAddrs []string, config *cluster.Config, topic, group string) (*Consumer, error) {
	//FIXME: consumer的config是否需要支持配置
	consumer, err := cluster.NewConsumer(brokerAddrs, group, []string{topic}, config)
	if err != nil {
		log.Errorf("kafka consumer init failed, addrs:%s, err:%v", brokerAddrs, err)
		return nil, err
	}
	go func() {
		for err := range consumer.Errors() {
			log.Warnf("consumer err : %v", err)
		}
	}()
	return &Consumer{
		topic:          topic,
		group:          group,
		consumer:       consumer,
		padding:        0,
		partitionHeads: make(map[int32]*ackNode),
		ackMessages:    make(map[int32]map[int64]*ackNode),
	}, nil
}
Example #6
0
func (m *Metadata) RefreshMetadata() error {
	queueConfigs := make(map[string]QueueConfig)

	err := m.manager.RefreshMetadata()
	if err != nil {
		return errors.Trace(err)
	}

	queues, _, err := m.zkClient.Children(m.queuePath)
	if err != nil {
		return errors.Trace(err)
	}

	for _, queue := range queues {
		_, stat, err := m.zkClient.Get(m.buildQueuePath(queue))
		if err != nil {
			log.Errorf("refresh err : %s", err)
			return errors.Trace(err)
		}

		exist, err := m.manager.ExistTopic(queue)
		if err != nil {
			log.Errorf("refresh err : %s", err)
			return errors.Trace(err)
		}
		if !exist {
			log.Errorf("queue : %q has metadata, but has no topic")
			continue
		}

		queueConfigs[queue] = QueueConfig{
			Queue:  queue,
			Ctime:  stat.Ctime / 1e3,
			Length: 0,
			Groups: make(map[string]GroupConfig),
		}
	}

	groupKeys, _, err := m.zkClient.Children(m.groupConfigPath)
	if err != nil {
		return errors.Trace(err)
	}

	for _, groupKey := range groupKeys {
		tokens := strings.Split(groupKey, ".")
		if len(tokens) != 2 {
			continue
		}
		queueName, groupName := tokens[1], tokens[0]
		queue, ok := queueConfigs[queueName]
		if !ok {
			continue
		}

		groupDataPath := fmt.Sprintf("%s/%s", m.groupConfigPath, groupKey)
		groupData, _, err := m.zkClient.Get(groupDataPath)
		if err != nil {
			log.Warnf("get %s err: %s", groupDataPath, err)
			continue
		}

		groupConfig := GroupConfig{}
		err = json.Unmarshal(groupData, &groupConfig)
		if err != nil {
			log.Warnf("Unmarshal %s data err: %s", groupDataPath, err)
			continue
		}

		queue.Groups[groupName] = groupConfig
	}

	m.mu.Lock()
	m.queueConfigs = queueConfigs
	m.mu.Unlock()
	return nil
}