Exemplo n.º 1
0
func processGroup(command string, groupId string, topic string) {
	partitions, err := kafkaClient.Partitions(topic)
	if err != nil {
		exit(err)
	}
	offsetManager, err := sarama.NewOffsetManagerFromClient(groupId, kafkaClient)
	if err != nil {
		exit(err)
	}
	var lag int64
	var timestamps = make([]int64, len(partitions))
	var controlChannel = make(chan []int64)
	for _, partition := range partitions {
		go processPartition(command, topic, partition, controlChannel, offsetManager)
	}
	for _, partition := range partitions {
		response := <-controlChannel
		lag += response[0]
		if command == "lag_and_time" {
			timestamps[partition] = response[1]
		}
	}
	if command == "lag_and_time" {
		max := timestamps[0]
		for _, value := range timestamps {
			if value > max {
				max = value
			}
		}
		timelag := time.Now().Unix() - max
		fmt.Println(groupId, topic, lag, timelag)
	} else {
		fmt.Println(groupId, topic, lag)
	}
}
Exemplo n.º 2
0
func processGroup(groupId string, topic string) {
	partitions, err := kafkaClient.Partitions(topic)
	if err != nil {
		exit(err)
	}
	offsetManager, err := sarama.NewOffsetManagerFromClient(groupId, kafkaClient)
	if err != nil {
		exit(err)
	}
	var lag int64
	var controlChannel = make(chan int64)
	for _, partition := range partitions {
		go processPartition(topic, partition, controlChannel, offsetManager)
	}
	for _ = range partitions {
		partitionLag := <-controlChannel
		lag += partitionLag
	}
	fmt.Println(topic, groupId, lag)
}
Exemplo n.º 3
0
func Consume() {
	config := sarama.NewConfig()
	//config.ChannelBufferSize =
	master, _ := sarama.NewClient([]string{"localhost:9095"}, config)
	cmas, _ := sarama.NewConsumerFromClient(master)
	offsetmanager, _ := sarama.NewOffsetManagerFromClient("test2", master)
	part_man, _ := offsetmanager.ManagePartition("tt.test", 1)
	//part_man.MarkOffset(0, "tt.test")
	off1, mata1 := part_man.NextOffset()
	fmt.Println(off1)
	fmt.Println(mata1)
	csm, _ := cmas.ConsumePartition("tt.test", 1, off1)
	//csm.Messages
	//Loop:
	for i := 0; i < 200; i++ {
		select {
		case message := <-csm.Messages():
			off1++
			part_man.MarkOffset(off1, "tt.test")
			fmt.Println(message)
		case err := <-csm.Errors():
			fmt.Println(err)
			//csm.Close()
			//			break Loop
		}
	}
	part_man.MarkOffset(off1, "tt.test")
	csm.Close()
	fmt.Println(off1)
	part_man.Close()
	//part_man2, _ := offsetmanager.ManagePartition("syslog-ng", 0)
	//off2, mata2 := part_man2.NextOffset()
	//part_man.MarkOffset(off2, "syslog-ng")
	//fmt.Println(off2)
	//fmt.Println(mata2)
	cmas.Close()
	master.Close()
}
Exemplo n.º 4
0
Arquivo: offset.go Projeto: fgeller/kt
func offsetRun(closer chan struct{}) {
	var err error
	if config.offset.verbose {
		sarama.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	conf := sarama.NewConfig()
	conf.Version = config.offset.version
	u, err := user.Current()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to read current user err=%v", err)
	}
	conf.ClientID = "kt-offset-" + u.Username
	if config.offset.verbose {
		fmt.Fprintf(os.Stderr, "sarama client configuration %#v\n", conf)
	}

	client, err := sarama.NewClient(config.offset.brokers, conf)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create client err=%v\n", err)
		os.Exit(1)
	}
	defer client.Close()

	om, err := sarama.NewOffsetManagerFromClient(config.offset.group, client)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create offset manager for group=%s err=%v\n", config.offset.group, err)
		os.Exit(1)
	}

	var wg sync.WaitGroup
	wg.Add(1)

	out := make(chan string)
	done := make(chan bool)
	go func(out chan string, done chan bool) {
		topics, err := client.Topics()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to read topics err=%v\n", err)
			os.Exit(1)
		}

		for _, t := range topics {
			if config.offset.args.topic == "" || config.offset.topic.MatchString(t) {
				offsetsForTopic(
					client.Coordinator,
					client.Partitions,
					client.GetOffset,
					om.ManagePartition,
					config.offset.group,
					t,
					out,
				)
			}
		}

		done <- true
	}(out, done)

printLoop:
	for {
		select {
		case m := <-out:
			fmt.Println(m)
		case <-done:
			break printLoop
		case <-closer:
			break printLoop
		}
	}
}