Example #1
0
func (m *Manager) GetTopics() ([]string, error) {
	topics, err := m.client.Topics()
	if err != nil {
		log.Warnf("get topics err : %s", err)
	}
	return topics, err
}
Example #2
0
func (c *httpClient) Send(uri string, data []byte) (err error) {
	log.Info(string(data))
	req, err := http.NewRequest("POST", uri, bytes.NewReader(data))
	if err != nil {
		log.Warnf("new http request err: %v", err)
		return err
	}
	resp, err := c.cli.Do(req)
	if err != nil {
		log.Warnf("do http request err: %v", err)
		return err
	}
	defer resp.Body.Close()

	respData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Warnf("do http request err: %v", err)
		return err
	}
	// TODO check
	_ = respData
	return
}
Example #3
0
func (m *RoamClient) Send(key string, data []byte) (err error) {
	if len(data) == 0 || string(data) == "[]" {
		return
	}
	sts, err := transToRoamStruct(data)
	if err != nil {
		log.Warnf("store profile log err : %v", err)
		return err
	}
	for i := range sts {
		log.Profile("%s", sts[i])
	}
	return
}
Example #4
0
func Add(key string, args ...int64) {
	var pkt *Packet
	if len(args) == 1 {
		pkt = &Packet{INCR, key, args[0], 0, 0}
	} else if len(args) == 2 {
		pkt = &Packet{INCR_EX, key, args[0], args[1], 0}
	} else if len(args) == 3 {
		pkt = &Packet{INCR_EX2, key, args[0], args[1], args[2]}
	}

	select {
	case defaultClient.in <- pkt:
	default:
		log.Warnf("metrics chan is full: %s", pkt)
	}
}
Example #5
0
func (s *Server) msgReceive(queue string, group string) string {
	var result string
	id, data, _, err := s.queue.RecvMessage(queue, group)
	if err != nil {
		log.Debugf("msgReceive failed: %s", errors.ErrorStack(err))
		result = err.Error()
	} else {
		err = s.queue.AckMessage(queue, group, id)
		if err != nil {
			log.Warnf("ack message queue:%q group:%q id:%q err:%s", queue, group, id, err)
			result = err.Error()
		} else {
			result = `{"action":"receive","msg":"` + string(data) + `"}`
		}
	}
	return result
}
Example #6
0
func transToRoamStruct(data []byte) (jsonStrs []string, err error) {
	var results []*MetricsStat
	err = json.Unmarshal(data, &results)
	if err != nil {
		return nil, err
	}

	action := func(st *MetricsStat) []*RoamStruct {
		ret := make([]*RoamStruct, 0, 2)
		actions := []string{SENT, RECV}
		for _, act := range actions {
			rst := &RoamStruct{
				Type:   WQS,
				Queue:  st.Queue,
				Group:  st.Group,
				Action: act,
			}
			switch act {
			case SENT:
				rst.Total = st.Sent.Total
				rst.AvgTime = st.Sent.Elapsed
			case RECV:
				rst.Total = st.Recv.Total
				rst.AvgTime = st.Recv.Elapsed
			}
			ret = append(ret, rst)
		}
		return ret
	}

	for _, st := range results {
		rsts := action(st)
		for _, rst := range rsts {
			stData, err := json.Marshal(rst)
			if err != nil {
				log.Warnf("transToRoamStruct err : %v", err)
				continue
			}
			jsonStrs = append(jsonStrs, string(stData))
		}
	}
	return
}
Example #7
0
func (ms *McServer) connLoop(conn net.Conn) {
	defer func(conn net.Conn) {
		log.Debugf("mc client closed :%s", conn.RemoteAddr())
		ms.mu.Lock()
		delete(ms.connPool, conn)
		ms.mu.Unlock()
		conn.Close()
		if err := recover(); err != nil {
			log.Errorf("mc connLoop panic error: %s", err)
		}
	}(conn)

	br := bufio.NewReaderSize(conn, ms.recvBuffSize)
	bw := bufio.NewWriterSize(conn, ms.sendBuffSize)

	for atomic.LoadInt32(&ms.stopping) == 0 {
		data, err := br.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				return
			}
			log.Warnf("mc server ReadLine err:%s", err)
			return
		}

		tokens := strings.Split(strings.TrimSpace(data), " ")
		cmd := tokens[0]
		command, exists := commands[cmd]
		if !exists {
			command = commandUnkown
			cmd = "unsupported"
		}
		metrics.Add(cmd, 1)
		err = command(ms.queue, tokens, br, bw)
		bw.Flush()
		if err != nil {
			//command返回错误一定是不能容忍的错误,需要退出循环关闭连接,防止将后续有效数据的格式都破坏掉
			log.Errorf("mc bad command:%s", errors.ErrorStack(err))
			return
		}
	}
}
Example #8
0
func (m *Monitor) storeStatistic() {
	t := time.Now().Unix() / 10 * 10
	time := strconv.FormatInt(int64(t), 10)
	snapshot := make(map[string]int64)
	//reduce hoding mutex druation
	m.mu.Lock()
	for k, v := range m.statisticMap {
		snapshot[k] = v
		m.statisticMap[k] = 0
	}
	m.mu.Unlock()

	for k, v := range snapshot {
		key := k + time
		_, err := m.redisClient.IncrBy(key, v).Result()
		if err != nil {
			log.Warnf("storeStatistic HIncrBy err: %s", err)
		}
	}
}
Example #9
0
func NewConsumer(brokerAddrs []string, config *cluster.Config, topic, group string) (*Consumer, error) {
	//FIXME: consumer的config是否需要支持配置
	consumer, err := cluster.NewConsumer(brokerAddrs, group, []string{topic}, config)
	if err != nil {
		log.Errorf("kafka consumer init failed, addrs:%s, err:%v", brokerAddrs, err)
		return nil, err
	}
	go func() {
		for err := range consumer.Errors() {
			log.Warnf("consumer err : %v", err)
		}
	}()
	return &Consumer{
		topic:          topic,
		group:          group,
		consumer:       consumer,
		padding:        0,
		partitionHeads: make(map[int32]*ackNode),
		ackMessages:    make(map[int32]map[int64]*ackNode),
	}, nil
}
Example #10
0
func NewMetadata(config *config.Config, sconfig *sarama.Config) (*Metadata, error) {
	zkClient, err := zookeeper.NewZkClient(strings.Split(config.MetaDataZKAddr, ","))
	if err != nil {
		return nil, errors.Trace(err)
	}

	zkRoot := config.MetaDataZKRoot
	if strings.EqualFold(zkRoot, root) {
		zkRoot = ""
	}

	groupConfigPath := fmt.Sprintf("%s%s", zkRoot, groupConfigPathSuffix)
	queuePath := fmt.Sprintf("%s%s", zkRoot, queuePathSuffix)
	servicePath := fmt.Sprintf("%s%s", zkRoot, servicePathPrefix)
	operationPath := fmt.Sprintf("%s%s", zkRoot, operationPathPrefix)

	err = zkClient.CreateRec(groupConfigPath, "", 0)
	if err != nil && err != zk.ErrNodeExists {
		return nil, errors.Trace(err)
	}

	err = zkClient.CreateRec(queuePath, "", 0)
	if err != nil && err != zk.ErrNodeExists {
		return nil, errors.Trace(err)
	}

	err = zkClient.CreateRec(servicePath, "", 0)
	if err != nil && err != zk.ErrNodeExists {
		return nil, errors.Trace(err)
	}

	err = zkClient.CreateRec(operationPath, "", 0)
	if err != nil && err != zk.ErrNodeExists {
		return nil, errors.Trace(err)
	}

	manager, err := kafka.NewManager(strings.Split(config.KafkaZKAddr, ","), config.KafkaZKRoot, sconfig)
	if err != nil {
		return nil, errors.Trace(err)
	}

	metadata := &Metadata{
		config:          config,
		zkClient:        zkClient,
		manager:         manager,
		groupConfigPath: groupConfigPath,
		queuePath:       queuePath,
		servicePath:     servicePath,
		operationPath:   operationPath,
		queueConfigs:    make(map[string]QueueConfig),
		closeCh:         make(chan struct{}),
	}

	err = metadata.RefreshMetadata()
	if err != nil {
		return nil, errors.Trace(err)
	}

	go func(m *Metadata) {
		timeout := time.NewTicker(sconfig.Metadata.RefreshFrequency)
		for {
			select {
			case <-timeout.C:
				err := m.RefreshMetadata()
				if err != nil {
					log.Warnf("timeout refresh metadata err : %s", err)
				}
			case <-m.closeCh:
				timeout.Stop()
				return
			}
		}
	}(metadata)

	return metadata, nil
}
Example #11
0
func (m *Metadata) RefreshMetadata() error {
	queueConfigs := make(map[string]QueueConfig)

	err := m.manager.RefreshMetadata()
	if err != nil {
		return errors.Trace(err)
	}

	queues, _, err := m.zkClient.Children(m.queuePath)
	if err != nil {
		return errors.Trace(err)
	}

	for _, queue := range queues {
		_, stat, err := m.zkClient.Get(m.buildQueuePath(queue))
		if err != nil {
			log.Errorf("refresh err : %s", err)
			return errors.Trace(err)
		}

		exist, err := m.manager.ExistTopic(queue)
		if err != nil {
			log.Errorf("refresh err : %s", err)
			return errors.Trace(err)
		}
		if !exist {
			log.Errorf("queue : %q has metadata, but has no topic")
			continue
		}

		queueConfigs[queue] = QueueConfig{
			Queue:  queue,
			Ctime:  stat.Ctime / 1e3,
			Length: 0,
			Groups: make(map[string]GroupConfig),
		}
	}

	groupKeys, _, err := m.zkClient.Children(m.groupConfigPath)
	if err != nil {
		return errors.Trace(err)
	}

	for _, groupKey := range groupKeys {
		tokens := strings.Split(groupKey, ".")
		if len(tokens) != 2 {
			continue
		}
		queueName, groupName := tokens[1], tokens[0]
		queue, ok := queueConfigs[queueName]
		if !ok {
			continue
		}

		groupDataPath := fmt.Sprintf("%s/%s", m.groupConfigPath, groupKey)
		groupData, _, err := m.zkClient.Get(groupDataPath)
		if err != nil {
			log.Warnf("get %s err: %s", groupDataPath, err)
			continue
		}

		groupConfig := GroupConfig{}
		err = json.Unmarshal(groupData, &groupConfig)
		if err != nil {
			log.Warnf("Unmarshal %s data err: %s", groupDataPath, err)
			continue
		}

		queue.Groups[groupName] = groupConfig
	}

	m.mu.Lock()
	m.queueConfigs = queueConfigs
	m.mu.Unlock()
	return nil
}