func (this *Clusters) clusterSummary(zkcluster *zk.ZkCluster) (brokers, topics, partitions int, flat, cum int64) { brokerInfos := zkcluster.Brokers() brokers = len(brokerInfos) kfk, err := sarama.NewClient(zkcluster.BrokerList(), saramaConfig()) if err != nil { this.Ui.Error(err.Error()) return } defer kfk.Close() topicInfos, _ := kfk.Topics() topics = len(topicInfos) for _, t := range topicInfos { alivePartitions, _ := kfk.WritablePartitions(t) partitions += len(alivePartitions) for _, partitionID := range alivePartitions { latestOffset, _ := kfk.GetOffset(t, partitionID, sarama.OffsetNewest) oldestOffset, _ := kfk.GetOffset(t, partitionID, sarama.OffsetOldest) flat += (latestOffset - oldestOffset) cum += latestOffset } } return }
func (this *Topics) addTopic(zkcluster *zk.ZkCluster, topic string, replicas, partitions int) error { this.Ui.Info(fmt.Sprintf("creating kafka topic: %s", topic)) ts := sla.DefaultSla() ts.Partitions = partitions ts.Replicas = replicas lines, err := zkcluster.AddTopic(topic, ts) if err != nil { return err } for _, l := range lines { this.Ui.Output(color.Yellow(l)) } if this.ipInNumber { this.Ui.Output(fmt.Sprintf("\tzookeeper.connect: %s", zkcluster.ZkConnectAddr())) this.Ui.Output(fmt.Sprintf("\t broker.list: %s", strings.Join(zkcluster.BrokerList(), ","))) } else { this.Ui.Output(fmt.Sprintf("\tzookeeper.connect: %s", zkcluster.NamedZkConnectAddr())) this.Ui.Output(fmt.Sprintf("\t broker.list: %s", strings.Join(zkcluster.NamedBrokerList(), ","))) } return nil }
func (this *Peek) consumeCluster(zkcluster *zk.ZkCluster, topicPattern string, partitionId int, msgChan chan *sarama.ConsumerMessage) { brokerList := zkcluster.BrokerList() if len(brokerList) == 0 { return } kfk, err := sarama.NewClient(brokerList, sarama.NewConfig()) if err != nil { this.Ui.Output(err.Error()) return } //defer kfk.Close() // FIXME how to close it topics, err := kfk.Topics() if err != nil { this.Ui.Output(err.Error()) return } for _, t := range topics { if patternMatched(t, topicPattern) { go this.simpleConsumeTopic(zkcluster, kfk, t, int32(partitionId), msgChan) } } }
func (this *Mirror) makePub(c2 *zk.ZkCluster) (sarama.AsyncProducer, error) { cf := sarama.NewConfig() cf.Metadata.RefreshFrequency = time.Minute * 10 cf.Metadata.Retry.Max = 3 cf.Metadata.Retry.Backoff = time.Second * 3 cf.ChannelBufferSize = 1000 cf.Producer.Return.Errors = true cf.Producer.Flush.Messages = 2000 // 2000 message in batch cf.Producer.Flush.Frequency = time.Second // flush interval cf.Producer.Flush.MaxMessages = 0 // unlimited cf.Producer.RequiredAcks = sarama.WaitForLocal cf.Producer.Retry.Backoff = time.Second * 4 cf.Producer.Retry.Max = 3 cf.Net.DialTimeout = time.Second * 30 cf.Net.WriteTimeout = time.Second * 30 cf.Net.ReadTimeout = time.Second * 30 switch this.Compress { case "gzip": cf.Producer.Compression = sarama.CompressionGZIP case "snappy": cf.Producer.Compression = sarama.CompressionSnappy } return sarama.NewAsyncProducer(c2.BrokerList(), cf) }
func (this *Topics) clusterSummary(zkcluster *zk.ZkCluster) []topicSummary { r := make([]topicSummary, 0, 10) kfk, err := sarama.NewClient(zkcluster.BrokerList(), saramaConfig()) if err != nil { this.Ui.Error(err.Error()) return nil } defer kfk.Close() topicInfos, _ := kfk.Topics() for _, t := range topicInfos { flat := int64(0) cum := int64(0) alivePartitions, _ := kfk.WritablePartitions(t) for _, partitionID := range alivePartitions { latestOffset, _ := kfk.GetOffset(t, partitionID, sarama.OffsetNewest) oldestOffset, _ := kfk.GetOffset(t, partitionID, sarama.OffsetOldest) flat += (latestOffset - oldestOffset) cum += latestOffset } r = append(r, topicSummary{zkcluster.ZkZone().Name(), zkcluster.Name(), t, len(alivePartitions), flat, cum}) } return r }
func (this *Mirror) makePub(c2 *zk.ZkCluster) (sarama.AsyncProducer, error) { // TODO setup batch size cf := sarama.NewConfig() switch this.compress { case "gzip": cf.Producer.Compression = sarama.CompressionGZIP case "snappy": cf.Producer.Compression = sarama.CompressionSnappy } return sarama.NewAsyncProducer(c2.BrokerList(), cf) }
func (this *TopBroker) clusterTopProducers(zkcluster *zk.ZkCluster) { kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig()) if err != nil { return } defer kfk.Close() for { hostOffsets := make(map[string]int64) topics, err := kfk.Topics() swallow(err) <-this.signalsCh[zkcluster.Name()] for _, topic := range topics { if !patternMatched(topic, this.topic) { continue } partions, err := kfk.WritablePartitions(topic) swallow(err) for _, partitionID := range partions { leader, err := kfk.Leader(topic, partitionID) swallow(err) latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) swallow(err) host, _, err := net.SplitHostPort(leader.Addr()) swallow(err) if this.shortIp { host = shortIp(host) } if _, present := hostOffsets[host]; !present { hostOffsets[host] = 0 } hostOffsets[host] += latestOffset } } this.hostOffsetCh <- hostOffsets kfk.RefreshMetadata(topics...) } }
func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) []string { brokerList := zkcluster.BrokerList() if len(brokerList) == 0 { this.Ui.Warn(fmt.Sprintf("%s empty brokers", zkcluster.Name())) return nil } kfk, err := sarama.NewClient(brokerList, saramaConfig()) if err != nil { this.Ui.Error(fmt.Sprintf("%s %+v %s", zkcluster.Name(), brokerList, err.Error())) return nil } defer kfk.Close() topics, err := kfk.Topics() swallow(err) if len(topics) == 0 { return nil } lines := make([]string, 0, 10) for _, topic := range topics { // get partitions and check if some dead alivePartitions, err := kfk.WritablePartitions(topic) if err != nil { this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch writable partitions: %v", zkcluster.Name(), topic, err)) continue } partions, err := kfk.Partitions(topic) if err != nil { this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch partitions: %v", zkcluster.Name(), topic, err)) continue } if len(alivePartitions) != len(partions) { this.Ui.Error(fmt.Sprintf("%s topic[%s] has %s partitions: %+v/%+v", zkcluster.Name(), topic, color.Red("dead"), alivePartitions, partions)) } for _, partitionID := range alivePartitions { replicas, err := kfk.Replicas(topic, partitionID) if err != nil { this.Ui.Error(fmt.Sprintf("%s topic[%s] P:%d: %v", zkcluster.Name(), topic, partitionID, err)) continue } isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID) underReplicated := false if len(isr) != len(replicas) { underReplicated = true } if underReplicated { leader, err := kfk.Leader(topic, partitionID) swallow(err) latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) swallow(err) oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest) swallow(err) lines = append(lines, fmt.Sprintf("\t%s Partition:%d/%s Leader:%d Replicas:%+v Isr:%+v/%s Offset:%d-%d Num:%d", topic, partitionID, gofmt.PrettySince(partitionCtime), leader.ID(), replicas, isr, gofmt.PrettySince(isrMtime), oldestOffset, latestOffset, latestOffset-oldestOffset)) } } } return lines }
func (this *Topics) displayTopicsOfCluster(zkcluster *zk.ZkCluster) { echoBuffer := func(lines []string) { for _, l := range lines { this.Ui.Output(l) } } linesInTopicMode := make([]string, 0) if this.verbose { linesInTopicMode = this.echoOrBuffer(zkcluster.Name(), linesInTopicMode) } // get all alive brokers within this cluster brokers := zkcluster.Brokers() if len(brokers) == 0 { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s", " ", color.Red("%s empty brokers", zkcluster.Name())), linesInTopicMode) echoBuffer(linesInTopicMode) return } if this.verbose { sortedBrokerIds := make([]string, 0, len(brokers)) for brokerId, _ := range brokers { sortedBrokerIds = append(sortedBrokerIds, brokerId) } sort.Strings(sortedBrokerIds) for _, brokerId := range sortedBrokerIds { if this.ipInNumber { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ", color.Green(brokerId), brokers[brokerId]), linesInTopicMode) } else { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ", color.Green(brokerId), brokers[brokerId].NamedString()), linesInTopicMode) } } } kfk, err := sarama.NewClient(zkcluster.BrokerList(), saramaConfig()) if err != nil { if this.verbose { linesInTopicMode = this.echoOrBuffer(color.Yellow("%5s%+v %s", " ", zkcluster.BrokerList(), err.Error()), linesInTopicMode) } return } defer kfk.Close() topics, err := kfk.Topics() swallow(err) if len(topics) == 0 { if this.topicPattern == "" && this.verbose { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%5s%s", " ", color.Magenta("no topics")), linesInTopicMode) echoBuffer(linesInTopicMode) } return } sortedTopics := make([]string, 0, len(topics)) for _, t := range topics { sortedTopics = append(sortedTopics, t) } sort.Strings(sortedTopics) topicsCtime := zkcluster.TopicsCtime() hasTopicMatched := false for _, topic := range sortedTopics { if !patternMatched(topic, this.topicPattern) { continue } if this.since > 0 && time.Since(topicsCtime[topic]) > this.since { continue } this.topicN++ hasTopicMatched = true if this.verbose { linesInTopicMode = this.echoOrBuffer(strings.Repeat(" ", 4)+color.Cyan(topic), linesInTopicMode) } // get partitions and check if some dead alivePartitions, err := kfk.WritablePartitions(topic) swallow(err) partions, err := kfk.Partitions(topic) swallow(err) if len(alivePartitions) != len(partions) { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %s P: %s/%+v", zkcluster.Name(), color.Cyan("%-50s", topic), color.Red("partial dead"), color.Green("%+v", alivePartitions), partions), linesInTopicMode) } replicas, err := kfk.Replicas(topic, partions[0]) if err != nil { this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partions[0], err)) } this.partitionN += len(partions) if !this.verbose { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %3dP %dR %s", zkcluster.Name(), color.Cyan("%-50s", topic), len(partions), len(replicas), gofmt.PrettySince(topicsCtime[topic])), linesInTopicMode) continue } for _, partitionID := range alivePartitions { leader, err := kfk.Leader(topic, partitionID) swallow(err) replicas, err := kfk.Replicas(topic, partitionID) if err != nil { this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partitionID, err)) } isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID) isrMtimeSince := gofmt.PrettySince(isrMtime) if time.Since(isrMtime).Hours() < 24 { // ever out of sync last 24h isrMtimeSince = color.Magenta(isrMtimeSince) } underReplicated := false if len(isr) != len(replicas) { underReplicated = true } latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) swallow(err) oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest) swallow(err) if this.count > 0 && (latestOffset-oldestOffset) < this.count { continue } this.totalMsgs += latestOffset - oldestOffset this.totalOffsets += latestOffset if !underReplicated { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%+v Offset:%16s - %-16s Num:%-15s %s-%s", partitionID, color.Green("%d", leader.ID()), replicas, isr, gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset), gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode) } else { // use red for alert linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%s Offset:%16s - %-16s Num:%-15s %s-%s", partitionID, color.Green("%d", leader.ID()), replicas, color.Red("%+v", isr), gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset), gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode) } } } if this.topicPattern != "" { if hasTopicMatched { echoBuffer(linesInTopicMode) } } else { echoBuffer(linesInTopicMode) } }
func (this *Top) clusterTopProducers(zkcluster *zk.ZkCluster) { cluster := zkcluster.Name() brokerList := zkcluster.BrokerList() if len(brokerList) == 0 { return } kfk, err := sarama.NewClient(brokerList, sarama.NewConfig()) if err != nil { return } defer kfk.Close() for { topics, err := kfk.Topics() if err != nil || len(topics) == 0 { return } for _, topic := range topics { if !patternMatched(topic, this.topicPattern) { continue } msgs := int64(0) alivePartitions, err := kfk.WritablePartitions(topic) swallow(err) for _, partitionID := range alivePartitions { latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) if err != nil { // this broker is down continue } msgs += latestOffset } this.mu.Lock() if _, present := this.brokers[cluster+":"+topic]; !present { // calculate the broker leading partitions leadingPartitions := make(map[string]int) // broker:lead partitions n for _, pid := range alivePartitions { leader, err := kfk.Leader(topic, pid) swallow(err) leadingPartitions[leader.Addr()]++ } brokers := make([]string, 0) for addr, n := range leadingPartitions { brokers = append(brokers, fmt.Sprintf("%d@%s", n, addr)) } this.brokers[cluster+":"+topic] = this.discardPortOfBrokerAddr(brokers) } this.counters[cluster+":"+topic] = float64(msgs) this.partitions[cluster+":"+topic] = len(alivePartitions) this.mu.Unlock() } time.Sleep(time.Second) kfk.RefreshMetadata(topics...) } }