func processGroup(command string, groupId string, topic string) { partitions, err := kafkaClient.Partitions(topic) if err != nil { exit(err) } offsetManager, err := sarama.NewOffsetManagerFromClient(groupId, kafkaClient) if err != nil { exit(err) } var lag int64 var timestamps = make([]int64, len(partitions)) var controlChannel = make(chan []int64) for _, partition := range partitions { go processPartition(command, topic, partition, controlChannel, offsetManager) } for _, partition := range partitions { response := <-controlChannel lag += response[0] if command == "lag_and_time" { timestamps[partition] = response[1] } } if command == "lag_and_time" { max := timestamps[0] for _, value := range timestamps { if value > max { max = value } } timelag := time.Now().Unix() - max fmt.Println(groupId, topic, lag, timelag) } else { fmt.Println(groupId, topic, lag) } }
func processGroup(groupId string, topic string) { partitions, err := kafkaClient.Partitions(topic) if err != nil { exit(err) } offsetManager, err := sarama.NewOffsetManagerFromClient(groupId, kafkaClient) if err != nil { exit(err) } var lag int64 var controlChannel = make(chan int64) for _, partition := range partitions { go processPartition(topic, partition, controlChannel, offsetManager) } for _ = range partitions { partitionLag := <-controlChannel lag += partitionLag } fmt.Println(topic, groupId, lag) }
func Consume() { config := sarama.NewConfig() //config.ChannelBufferSize = master, _ := sarama.NewClient([]string{"localhost:9095"}, config) cmas, _ := sarama.NewConsumerFromClient(master) offsetmanager, _ := sarama.NewOffsetManagerFromClient("test2", master) part_man, _ := offsetmanager.ManagePartition("tt.test", 1) //part_man.MarkOffset(0, "tt.test") off1, mata1 := part_man.NextOffset() fmt.Println(off1) fmt.Println(mata1) csm, _ := cmas.ConsumePartition("tt.test", 1, off1) //csm.Messages //Loop: for i := 0; i < 200; i++ { select { case message := <-csm.Messages(): off1++ part_man.MarkOffset(off1, "tt.test") fmt.Println(message) case err := <-csm.Errors(): fmt.Println(err) //csm.Close() // break Loop } } part_man.MarkOffset(off1, "tt.test") csm.Close() fmt.Println(off1) part_man.Close() //part_man2, _ := offsetmanager.ManagePartition("syslog-ng", 0) //off2, mata2 := part_man2.NextOffset() //part_man.MarkOffset(off2, "syslog-ng") //fmt.Println(off2) //fmt.Println(mata2) cmas.Close() master.Close() }
func offsetRun(closer chan struct{}) { var err error if config.offset.verbose { sarama.Logger = log.New(os.Stderr, "", log.LstdFlags) } conf := sarama.NewConfig() conf.Version = config.offset.version u, err := user.Current() if err != nil { fmt.Fprintf(os.Stderr, "Failed to read current user err=%v", err) } conf.ClientID = "kt-offset-" + u.Username if config.offset.verbose { fmt.Fprintf(os.Stderr, "sarama client configuration %#v\n", conf) } client, err := sarama.NewClient(config.offset.brokers, conf) if err != nil { fmt.Fprintf(os.Stderr, "Failed to create client err=%v\n", err) os.Exit(1) } defer client.Close() om, err := sarama.NewOffsetManagerFromClient(config.offset.group, client) if err != nil { fmt.Fprintf(os.Stderr, "Failed to create offset manager for group=%s err=%v\n", config.offset.group, err) os.Exit(1) } var wg sync.WaitGroup wg.Add(1) out := make(chan string) done := make(chan bool) go func(out chan string, done chan bool) { topics, err := client.Topics() if err != nil { fmt.Fprintf(os.Stderr, "Failed to read topics err=%v\n", err) os.Exit(1) } for _, t := range topics { if config.offset.args.topic == "" || config.offset.topic.MatchString(t) { offsetsForTopic( client.Coordinator, client.Partitions, client.GetOffset, om.ManagePartition, config.offset.group, t, out, ) } } done <- true }(out, done) printLoop: for { select { case m := <-out: fmt.Println(m) case <-done: break printLoop case <-closer: break printLoop } } }