Example #1
0
func processPartition(topic string, partition int32, controlChannel chan int64, offsetManager sarama.OffsetManager) {
	pom, err := offsetManager.ManagePartition(topic, int32(partition))
	if err != nil {
		exit(err)
	}
	consumerOffset, _ := pom.NextOffset()
	offset, err := kafkaClient.GetOffset(topic, int32(partition), sarama.OffsetNewest)
	if err != nil {
		exit(err)
	}
	controlChannel <- offset - consumerOffset + 1
}
Example #2
0
func processPartition(command string, topic string, partition int32, controlChannel chan []int64, offsetManager sarama.OffsetManager) {
	pom, err := offsetManager.ManagePartition(topic, int32(partition))
	if err != nil {
		exit(err)
	}
	consumerOffset, _ := pom.NextOffset()
	offset, err := kafkaClient.GetOffset(topic, int32(partition), sarama.OffsetNewest)
	//fmt.Println(topic, partition, consumerOffset, offset)
	if err != nil {
		exit(err)
	}
	var response = make([]int64, 0)
	var timelag int64
	lag := offset - consumerOffset + 1
	response = append(response, lag)
	if command == "lag_and_time" {
		broker, err := kafkaClient.Leader(topic, partition)
		if err != nil {
			exit(err)
		}
		fetchRequest := sarama.FetchRequest{MaxWaitTime: 10000, MinBytes: 0}
		fetchRequest.AddBlock(topic, partition, consumerOffset-2, 500000)
		fetchResponse, err := broker.Fetch(&fetchRequest)
		if err != nil {
			exit(err)
		}
		block := fetchResponse.GetBlock(topic, partition)
		messages := block.MsgSet.Messages
		if len(messages) > 0 {
			msg := messages[0].Messages()[0].Msg.Value
			var decodedData map[string]interface{}
			codec.NewDecoderBytes(msg, &msgpack).Decode(&decodedData)
			timestamp := decodedData["timestamp"]
			switch timestamp := timestamp.(type) {
			case uint64:
				timelag = int64(timestamp)
			default:
				fmt.Println(timestamp)
				exit(errors.New("message is missing timestamp"))
			}
		} else {
			timelag = 0
		}
		response = append(response, timelag)
	}
	controlChannel <- response
}