func (this *Clusters) printSummary(zkzone *zk.ZkZone, clusterPattern string, port string) {
lines := []string{"Zone|Cluster|Brokers|Topics|Partitions|FlatMsg|Cum"}
type summary struct {
zone, cluster string
brokers, topics, partitions int
flat, cum int64
}
summaries := make([]summary, 0, 10)
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
if !patternMatched(zkcluster.Name(), clusterPattern) {
return
}
brokers, topics, partitions, flat, cum := this.clusterSummary(zkcluster)
summaries = append(summaries, summary{zkzone.Name(), zkcluster.Name(), brokers, topics, partitions,
flat, cum})
})
sortutil.DescByField(summaries, "cum")
var totalFlat, totalCum int64
for _, s := range summaries {
lines = append(lines, fmt.Sprintf("%s|%s|%d|%d|%d|%s|%s",
s.zone, s.cluster, s.brokers, s.topics, s.partitions,
gofmt.Comma(s.flat), gofmt.Comma(s.cum)))
totalCum += s.cum
totalFlat += s.flat
}
this.Ui.Output(columnize.SimpleFormat(lines))
this.Ui.Output(fmt.Sprintf("Flat:%s Cum:%s", gofmt.Comma(totalFlat), gofmt.Comma(totalCum)))
}
func (this *Clusters) printRegisteredBrokers(zkzone *zk.ZkZone) {
this.Ui.Output(zkzone.Name())
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
info := zkcluster.RegisteredInfo()
this.Ui.Output(fmt.Sprintf(" %s(%s)", info.Name(), info.Nickname))
registeredBrokers := info.Roster
if len(registeredBrokers) == 0 {
this.Ui.Warn(" brokers not registered")
} else {
for _, b := range registeredBrokers {
if this.ipInNumber {
this.Ui.Output(fmt.Sprintf(" %2d %s", b.Id, b.Addr()))
} else {
this.Ui.Output(fmt.Sprintf(" %2d %s", b.Id, b.NamedAddr()))
}
}
}
})
}
func (this *Consumers) printConsumersByHost(zkzone *zk.ZkZone, clusterPattern string) {
outputs := make(map[string]map[string]map[string]int) // host: {cluster: {topic: count}}
this.Ui.Output(color.Blue(zkzone.Name()))
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
if !patternMatched(zkcluster.Name(), clusterPattern) {
return
}
consumerGroups := zkcluster.ConsumerGroups()
for _, group := range consumerGroups {
for _, c := range group {
if _, present := outputs[c.Host()]; !present {
outputs[c.Host()] = make(map[string]map[string]int)
}
if _, present := outputs[c.Host()][zkcluster.Name()]; !present {
outputs[c.Host()][zkcluster.Name()] = make(map[string]int)
}
for topic, count := range c.Subscription {
outputs[c.Host()][zkcluster.Name()][topic] += count
}
}
}
})
sortedHosts := make([]string, 0, len(outputs))
for host, _ := range outputs {
sortedHosts = append(sortedHosts, host)
}
sort.Strings(sortedHosts)
for _, host := range sortedHosts {
tc := outputs[host]
this.Ui.Output(fmt.Sprintf("%s %+v", color.Green("%22s", host), tc))
}
}
func (this *Topics) printSummary(zkzone *zk.ZkZone, clusterPattern string) {
lines := []string{"Zone|Cluster|Topic|Partitions|FlatMsg|Cum"}
var totalFlat, totalCum int64
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
if !patternMatched(zkcluster.Name(), clusterPattern) {
return
}
summaries := this.clusterSummary(zkcluster)
sortutil.DescByField(summaries, "cum")
for _, s := range summaries {
lines = append(lines, fmt.Sprintf("%s|%s|%s|%d|%s|%s",
s.zone, s.cluster, s.topic, s.partitions, gofmt.Comma(s.flat), gofmt.Comma(s.cum)))
totalCum += s.cum
totalFlat += s.flat
}
})
this.Ui.Output(columnize.SimpleFormat(lines))
this.Ui.Output(fmt.Sprintf("Flat:%s Cum:%s", gofmt.Comma(totalFlat), gofmt.Comma(totalCum)))
}
func (this *Clusters) printClusters(zkzone *zk.ZkZone, clusterPattern string, port string) {
if this.registeredBrokers {
this.printRegisteredBrokers(zkzone)
return
}
type clusterInfo struct {
name, path string
nickname string
topicN, partitionN int
err string
priority int
public bool
retention int
replicas int
brokerInfos []zk.BrokerInfo
}
clusters := make([]clusterInfo, 0)
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
if !patternMatched(zkcluster.Name(), clusterPattern) {
return
}
ci := clusterInfo{
name: zkcluster.Name(),
path: zkcluster.Chroot(),
}
if this.neat {
clusters = append(clusters, ci)
return
}
// verbose mode, will calculate topics and partition count
brokerList := zkcluster.BrokerList()
if len(brokerList) == 0 {
ci.err = "no live brokers"
clusters = append(clusters, ci)
return
}
if port != "" {
for _, hostport := range brokerList {
_, p, err := net.SplitHostPort(hostport)
swallow(err)
if p != port {
return
}
}
}
info := zkcluster.RegisteredInfo()
if this.publicOnly && !info.Public {
return
}
if !this.verbose {
ci.brokerInfos = info.Roster
clusters = append(clusters, ci)
return
}
kfk, err := sarama.NewClient(brokerList, saramaConfig())
if err != nil {
ci.err = err.Error()
clusters = append(clusters, ci)
return
}
topics, err := kfk.Topics()
if err != nil {
ci.err = err.Error()
clusters = append(clusters, ci)
return
}
partitionN := 0
for _, topic := range topics {
partitions, err := kfk.Partitions(topic)
if err != nil {
ci.err = err.Error()
clusters = append(clusters, ci)
continue
}
partitionN += len(partitions)
}
clusters = append(clusters, clusterInfo{
name: zkcluster.Name(),
nickname: info.Nickname,
path: zkcluster.Chroot(),
topicN: len(topics),
partitionN: partitionN,
retention: info.Retention,
public: info.Public,
replicas: info.Replicas,
priority: info.Priority,
brokerInfos: info.Roster,
})
})
this.Ui.Output(fmt.Sprintf("%s: %d", zkzone.Name(), len(clusters)))
if this.verbose {
// 2 loop: 1. print the err clusters 2. print the good clusters
for _, c := range clusters {
if c.err == "" {
continue
}
this.Ui.Output(fmt.Sprintf("%30s: %s %s", c.name, c.path,
color.Red(c.err)))
}
// loop2
for _, c := range clusters {
if c.err != "" {
continue
}
this.Ui.Output(fmt.Sprintf("%30s: %s",
c.name, c.path))
brokers := []string{}
for _, broker := range c.brokerInfos {
if this.ipInNumber {
brokers = append(brokers, fmt.Sprintf("%d/%s:%d", broker.Id, broker.Host, broker.Port))
} else {
brokers = append(brokers, fmt.Sprintf("%d/%s", broker.Id, broker.NamedAddr()))
}
}
if len(brokers) > 0 {
sort.Strings(brokers)
this.Ui.Info(color.Green("%31s %s", " ", strings.Join(brokers, ", ")))
}
this.Ui.Output(strings.Repeat(" ", 4) +
color.Green("nick:%s public:%v topics:%d partitions:%d replicas:%d retention:%dh",
c.nickname, c.public,
c.topicN, c.partitionN, c.replicas, c.retention))
}
return
}
// not verbose mode
hostsWithoutDnsRecords := make([]string, 0)
for _, c := range clusters {
this.Ui.Output(fmt.Sprintf("%30s: %s", c.name, c.path))
brokers := []string{}
for _, broker := range c.brokerInfos {
if this.ipInNumber {
brokers = append(brokers, fmt.Sprintf("%d/%s:%d", broker.Id, broker.Host, broker.Port))
} else {
brokers = append(brokers, fmt.Sprintf("%d/%s", broker.Id, broker.NamedAddr()))
}
if broker.Addr() == broker.NamedAddr() {
hostsWithoutDnsRecords = append(hostsWithoutDnsRecords, fmt.Sprintf("%s:%s", c.name, broker.Addr()))
}
}
if len(brokers) > 0 {
sort.Strings(brokers)
this.Ui.Info(color.Green("%31s %s", " ", strings.Join(brokers, ", ")))
} else {
this.Ui.Warn(fmt.Sprintf("%31s no live registered brokers", " "))
}
}
if len(hostsWithoutDnsRecords) > 0 {
this.Ui.Warn("brokers without dns record:")
for _, broker := range hostsWithoutDnsRecords {
parts := strings.SplitN(broker, ":", 2)
this.Ui.Output(fmt.Sprintf("%30s: %s", parts[0], color.Yellow(parts[1])))
}
}
}
func (this *Topology) displayZoneTopology(zkzone *zk.ZkZone) {
this.Ui.Output(zkzone.Name())
// {cluster: {topic: brokerHostInfo}}
brokerInstances := make(map[string]map[string]*brokerHostInfo)
zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
if len(liveBrokers) == 0 {
this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster))
return
}
if this.cluster != "" && this.cluster != cluster {
return
}
brokerInstances[cluster] = make(map[string]*brokerHostInfo)
for _, broker := range liveBrokers {
if !patternMatched(broker.Host, this.hostPattern) {
continue
}
if _, present := brokerInstances[cluster][broker.Host]; !present {
brokerInstances[cluster][broker.Host] = newBrokerHostInfo()
}
brokerInstances[cluster][broker.Host].addPort(broker.Port, broker.Uptime())
}
// find how many partitions a broker is leading
zkcluster := zkzone.NewCluster(cluster)
brokerList := zkcluster.BrokerList()
if len(brokerList) == 0 {
this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster))
return
}
kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
if err != nil {
this.Ui.Error(color.Red(" %+v %s", brokerList, err.Error()))
return
}
topics, err := kfk.Topics()
swallow(err)
for _, topic := range topics {
partions, err := kfk.WritablePartitions(topic)
swallow(err)
for _, partitionID := range partions {
leader, err := kfk.Leader(topic, partitionID)
swallow(err)
host, _, err := net.SplitHostPort(leader.Addr())
swallow(err)
if !patternMatched(host, this.hostPattern) {
continue
}
latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
if err != nil {
this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err))
continue
}
oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
if err != nil {
this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err))
continue
}
brokerInstances[cluster][host].topicMsgs[topic] += (latestOffset - oldestOffset)
brokerInstances[cluster][host].addTopicPartition(topic, partitionID)
}
}
})
hosts := make(map[string]struct{})
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
for host, _ := range brokerInstances[zkcluster.Name()] {
hosts[host] = struct{}{}
}
})
sortedHosts := make([]string, 0)
for host, _ := range hosts {
sortedHosts = append(sortedHosts, host)
}
sort.Strings(sortedHosts)
// sort by host ip
sortedClusters := make([]string, 0, len(brokerInstances))
for c, _ := range brokerInstances {
sortedClusters = append(sortedClusters, c)
}
sort.Strings(sortedClusters)
portN := 0
hostN := 0
topicN := 0
partitionN := 0
for _, host := range sortedHosts {
tn := 0
pn := 0
mn := int64(0)
ports := make([]int, 0)
for _, cluster := range sortedClusters {
if _, present := brokerInstances[cluster][host]; !present {
continue
}
tn += len(brokerInstances[cluster][host].topicPartitions)
pn += brokerInstances[cluster][host].leadingPartitions()
mn += brokerInstances[cluster][host].totalMsgsInStock()
ports = append(ports, brokerInstances[cluster][host].tcpPorts...)
}
portN += len(ports)
topicN += tn
partitionN += pn
hostN += 1
this.Ui.Output(fmt.Sprintf(" %s leading: %2dT %3dP %15sM ports %2d:%+v",
color.Green("%15s", host),
tn,
pn,
gofmt.Comma(mn),
len(ports),
ports))
if this.verbose {
for _, cluster := range sortedClusters {
if _, present := brokerInstances[cluster][host]; !present {
continue
}
for _, tcpPort := range brokerInstances[cluster][host].tcpPorts {
this.Ui.Output(fmt.Sprintf("%40d %s", tcpPort,
gofmt.PrettySince(brokerInstances[cluster][host].uptimes[tcpPort])))
}
}
for _, cluster := range sortedClusters {
if _, present := brokerInstances[cluster][host]; !present {
continue
}
this.Ui.Output(color.Magenta("%30s", cluster))
for topic, partitions := range brokerInstances[cluster][host].topicPartitions {
this.Ui.Output(fmt.Sprintf("%40s: %15sM P%2d %+v",
topic,
gofmt.Comma(brokerInstances[cluster][host].topicMsgs[topic]),
len(partitions), partitions))
}
}
}
}
this.Ui.Output(fmt.Sprintf("%17s host:%d, topic:%d, partition:%d, instance:%d",
"-TOTAL-",
hostN, topicN, partitionN, portN))
}
func (this *Consumers) cleanupStaleConsumerGroups(zkzone *zk.ZkZone, clusterPattern string) {
// what consumer groups are safe to delete?
// 1. not online
// 2. have no offsets
this.Ui.Output(color.Blue(zkzone.Name()))
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
if !patternMatched(zkcluster.Name(), clusterPattern) {
return
}
this.Ui.Output(strings.Repeat(" ", 4) + zkcluster.Name())
consumerGroups := zkcluster.ConsumerGroups()
for group, consumers := range consumerGroups {
if len(consumers) > 0 {
// this consumer group is online
continue
}
if !patternMatched(group, this.groupPattern) {
continue
}
if !strings.HasPrefix(group, "console-consumer-") {
path := zkcluster.ConsumerGroupOffsetPath(group)
_, _, err := zkzone.Conn().Children(path)
if err == nil {
this.Ui.Warn(fmt.Sprintf("%s not empty, unsafe to cleanup", path))
continue
}
if err != gozk.ErrNoNode {
// should never happen
swallow(err)
}
}
// have no offsets, safe to delete
if this.confirmYes {
yes, err := this.Ui.Ask(fmt.Sprintf("confirm to remove cluster[%s] consumer group: %s? [Y/n]",
zkcluster.Name(), group))
swallow(err)
if strings.ToLower(yes) == "n" {
this.Ui.Info(fmt.Sprintf("%s skipped", group))
continue
}
} else {
yes, err := this.Ui.Ask(fmt.Sprintf("confirm to remove cluster[%s] consumer group: %s? [y/N]",
zkcluster.Name(), group))
swallow(err)
if strings.ToLower(yes) != "y" {
this.Ui.Info(fmt.Sprintf("%s skipped", group))
continue
}
}
// do delete this consumer group
zkzone.DeleteRecursive(zkcluster.ConsumerGroupRoot(group))
this.Ui.Info(fmt.Sprintf("%s deleted", group))
}
})
}
func (this *Consumers) printConsumersByGroupTable(zkzone *zk.ZkZone, clusterPattern string) {
lines := make([]string, 0)
header := "Zone|Cluster|M|Host|ConsumerGroup|Topic/Partition|Offset|Uptime"
lines = append(lines, header)
zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
groupTopicsMap := make(map[string]map[string]struct{}) // group:sub topics
if !patternMatched(zkcluster.Name(), clusterPattern) {
return
}
consumerGroups := zkcluster.ConsumerGroups()
sortedGroups := make([]string, 0, len(consumerGroups))
for group, _ := range consumerGroups {
if !patternMatched(group, this.groupPattern) {
continue
}
sortedGroups = append(sortedGroups, group)
}
sort.Strings(sortedGroups)
for _, group := range sortedGroups {
consumers := consumerGroups[group]
if this.onlineOnly && len(consumers) == 0 {
continue
}
if len(consumers) > 0 {
// sort by host
sortedIds := make([]string, 0)
consumersMap := make(map[string]*zk.ConsumerZnode)
for _, c := range consumers {
sortedIds = append(sortedIds, c.Id)
consumersMap[c.Id] = c
}
sort.Strings(sortedIds)
for _, consumerId := range sortedIds {
c := consumersMap[consumerId]
for topic, _ := range c.Subscription {
if !patternMatched(topic, this.topicPattern) {
continue
}
if len(groupTopicsMap[group]) == 0 {
groupTopicsMap[group] = make(map[string]struct{}, 5)
}
groupTopicsMap[group][topic] = struct{}{}
ownerByPartition := zkcluster.OwnersOfGroupByTopic(group, topic)
partitionsWithOffset := make(map[string]struct{})
for _, offset := range this.displayGroupOffsets(zkcluster, group, topic, false) {
onlineSymbol := "◉"
isOwner := false
if ownerByPartition[offset.partitionId] == consumerId {
onlineSymbol += "*" // owned by this consumer
isOwner = true
}
if this.ownerOnly && !isOwner {
continue
}
partitionsWithOffset[offset.partitionId] = struct{}{}
lines = append(lines,
fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s|%s",
zkzone.Name(), zkcluster.Name(),
onlineSymbol,
c.Host(),
group+"@"+c.Id[len(c.Id)-12:],
fmt.Sprintf("%s/%s", offset.topic, offset.partitionId),
offset.offset,
gofmt.PrettySince(c.Uptime())))
}
for partitionId, _ := range ownerByPartition {
if _, present := partitionsWithOffset[partitionId]; !present {
// this consumer is owner online, but has no offset
onlineSymbol := "◉"
isOwner := false
if ownerByPartition[partitionId] == consumerId {
onlineSymbol += "*"
isOwner = true
}
if this.ownerOnly && !isOwner {
continue
}
lines = append(lines,
fmt.Sprintf("%s|%s|%s|%s|%s|%s|?|%s",
zkzone.Name(), zkcluster.Name(),
onlineSymbol,
c.Host(),
group+"@"+c.Id[len(c.Id)-12:],
fmt.Sprintf("%s/%s", topic, partitionId),
gofmt.PrettySince(c.Uptime())))
}
}
}
}
} else {
// offline
for _, offset := range this.displayGroupOffsets(zkcluster, group, "", false) {
if !patternMatched(offset.topic, this.topicPattern) {
continue
}
lines = append(lines,
fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s|%s",
zkzone.Name(), zkcluster.Name(),
"◎",
" ",
group, fmt.Sprintf("%s/%s", offset.topic, offset.partitionId),
offset.offset, " "))
}
}
}
for group, topics := range groupTopicsMap {
if len(topics) > 1 {
// the same consumer group is consuming more than 1 topics
topicsLabel := make([]string, 0, len(topics))
for t, _ := range topics {
topicsLabel = append(topicsLabel, t)
}
this.Ui.Warn(fmt.Sprintf("%35s consuming: %+v", group, topicsLabel))
}
}
})
if !this.warnOnly {
this.Ui.Output(columnize.SimpleFormat(lines))
}
}