func (this *Stat) Run(args []string) (exitCode int) { cmdFlags := flag.NewFlagSet("stat", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "") if err := cmdFlags.Parse(args); err != nil { return 1 } if this.zone == "" { this.Ui.Error("unknown zone") return 2 } if len(args) == 0 { this.Ui.Error("missing path") return 2 } this.path = args[len(args)-1] zkzone := gzk.NewZkZone(gzk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) defer zkzone.Close() conn := zkzone.Conn() _, stat, err := conn.Get(this.path) must(err) this.Ui.Output(fmt.Sprintf("%# v", pretty.Formatter(*stat))) ctime := gzk.ZkTimestamp(stat.Ctime).Time() mtime := gzk.ZkTimestamp(stat.Mtime).Time() this.Ui.Output(fmt.Sprintf("ctime: %s, mtime: %s", gofmt.PrettySince(ctime), gofmt.PrettySince(mtime))) return }
// Print all controllers of all clusters within a zone. func (this *Controllers) printControllers(zkzone *zk.ZkZone) { this.Ui.Output(zkzone.Name()) zkzone.ForSortedControllers(func(cluster string, controller *zk.ControllerMeta) { if !patternMatched(cluster, this.cluster) { return } this.Ui.Output(strings.Repeat(" ", 4) + cluster) if controller == nil { this.Ui.Output(fmt.Sprintf("\t%s", color.Red("empty"))) } else { epochSince := time.Since(controller.Mtime.Time()) epochSinceStr := gofmt.PrettySince(controller.Mtime.Time()) if epochSince < time.Hour*2*24 { epochSinceStr = color.Red(epochSinceStr) } this.Ui.Output(fmt.Sprintf("\t%-2s %21s epoch:%2s/%-20s uptime:%s", controller.Broker.Id, controller.Broker.Addr(), controller.Epoch, epochSinceStr, gofmt.PrettySince(controller.Broker.Uptime()))) } }) }
func (this *Brokers) clusterBrokers(zone, cluster string, brokers map[string]*zk.BrokerZnode) []string { if !patternMatched(cluster, this.cluster) { return nil } if brokers == nil || len(brokers) == 0 { return []string{fmt.Sprintf("%s|%s|%s|%s|%s", zone, cluster, " ", color.Red("empty brokers"), " ")} } lines := make([]string, 0, len(brokers)) if this.staleOnly { // try each broker's aliveness for brokerId, broker := range brokers { cf := sarama.NewConfig() cf.Net.ReadTimeout = time.Second * 4 cf.Net.WriteTimeout = time.Second * 4 kfk, err := sarama.NewClient([]string{broker.Addr()}, cf) if err != nil { lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s", zone, cluster, brokerId, broker.Addr(), fmt.Sprintf("%s: %v", gofmt.PrettySince(broker.Uptime()), err))) } else { kfk.Close() } } return lines } // sort by broker id sortedBrokerIds := make([]string, 0, len(brokers)) for brokerId, _ := range brokers { sortedBrokerIds = append(sortedBrokerIds, brokerId) } sort.Strings(sortedBrokerIds) for _, brokerId := range sortedBrokerIds { b := brokers[brokerId] uptime := gofmt.PrettySince(b.Uptime()) if time.Since(b.Uptime()) < time.Hour*24*7 { uptime = color.Green(uptime) } if this.ipInNumber { lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s", zone, cluster, brokerId, b.Addr(), gofmt.PrettySince(b.Uptime()))) } else { lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s", zone, cluster, brokerId, b.NamedAddr(), gofmt.PrettySince(b.Uptime()))) } } return lines }
func (b BrokerZnode) NamedString() string { addr, _ := b.NamedAddr() return fmt.Sprintf("%s ver:%d uptime:%s", addr, b.Version, gofmt.PrettySince(b.Uptime())) }
func (this *Kguard) Run(args []string) (exitCode int) { cmdFlags := flag.NewFlagSet("kguard", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "") cmdFlags.BoolVar(&this.longFmt, "l", false, "") if err := cmdFlags.Parse(args); err != nil { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) kguards, err := zkzone.KguardInfos() if err != nil { this.Ui.Error(fmt.Sprintf("%s %v", zk.KguardLeaderPath, err.Error())) return } leader := kguards[0] this.Ui.Output(fmt.Sprintf("%s(out of %d candidates) up: %s", color.Green(leader.Host), leader.Candidates, gofmt.PrettySince(leader.Ctime))) if this.longFmt { this.showKguardVersion(leader.Host) this.showStats(leader.Host) } return }
func (this *Lags) printConsumersLag(zkcluster *zk.ZkCluster) { // sort by group name consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern) sortedGroups := make([]string, 0, len(consumersByGroup)) for group, _ := range consumersByGroup { sortedGroups = append(sortedGroups, group) } sort.Strings(sortedGroups) for _, group := range sortedGroups { lines := make([]string, 0, 100) sortedTopicAndPartitionIds := make([]string, 0) consumers := make(map[string]zk.ConsumerMeta) for _, t := range consumersByGroup[group] { key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId) sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key) consumers[key] = t } sort.Strings(sortedTopicAndPartitionIds) for _, topicAndPartitionId := range sortedTopicAndPartitionIds { consumer := consumers[topicAndPartitionId] if !patternMatched(consumer.Topic, this.topicPattern) { continue } var ( lagOutput string symbol string ) if consumer.Lag > int64(this.lagThreshold) { lagOutput = color.Red("%15s", gofmt.Comma(consumer.Lag)) if consumer.Online { symbol = color.Yellow("⚠︎︎") } else { symbol = color.Yellow("◎") } } else { lagOutput = color.Blue("%15s", gofmt.Comma(consumer.Lag)) if consumer.Online { symbol = color.Green("◉") } else { symbol = color.Yellow("◎") } } if consumer.Online { if this.problematicMode && consumer.Lag <= int64(this.lagThreshold) { continue } var ( host string uptime string ) if consumer.ConsumerZnode == nil { host = "unrecognized" uptime = "-" } else { host = color.Green("%s", consumer.ConsumerZnode.Host()) if time.Since(consumer.ConsumerZnode.Uptime()) < time.Hour { uptime = color.Magenta(gofmt.PrettySince(consumer.ConsumerZnode.Uptime())) } else { uptime = gofmt.PrettySince(consumer.ConsumerZnode.Uptime()) } } lines = append(lines, fmt.Sprintf("\t%s %35s/%-2s %12s -> %-15s %s %-10s %s %s", symbol, consumer.Topic, consumer.PartitionId, gofmt.Comma(consumer.ProducerOffset), gofmt.Comma(consumer.ConsumerOffset), lagOutput, gofmt.PrettySince(consumer.Mtime.Time()), host, uptime)) } else if !this.onlineOnly { lines = append(lines, fmt.Sprintf("\t%s %35s/%-2s %12s -> %-12s %s %s", symbol, consumer.Topic, consumer.PartitionId, gofmt.Comma(consumer.ProducerOffset), gofmt.Comma(consumer.ConsumerOffset), lagOutput, gofmt.PrettySince(consumer.Mtime.Time()))) } } if len(lines) > 0 { this.Ui.Output(strings.Repeat(" ", 4) + group) for _, l := range lines { this.Ui.Output(l) } } } }
func (this *Lags) printConsumersLagTable(zkcluster *zk.ZkCluster) { lines := make([]string, 0) header := "ConsumerGroup|Topic/Partition|Produced|Consumed|Lag|Committed|Uptime" lines = append(lines, header) // sort by group name consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern) sortedGroups := make([]string, 0, len(consumersByGroup)) for group, _ := range consumersByGroup { sortedGroups = append(sortedGroups, group) } sort.Strings(sortedGroups) for _, group := range sortedGroups { if !patternMatched(group, this.groupPattern) { continue } sortedTopicAndPartitionIds := make([]string, 0, len(consumersByGroup[group])) consumers := make(map[string]zk.ConsumerMeta) for _, t := range consumersByGroup[group] { key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId) sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key) consumers[key] = t } sort.Strings(sortedTopicAndPartitionIds) for _, topicAndPartitionId := range sortedTopicAndPartitionIds { consumer := consumers[topicAndPartitionId] if !patternMatched(consumer.Topic, this.topicPattern) { continue } if !consumer.Online { continue } if this.problematicMode && consumer.Lag <= int64(this.lagThreshold) { continue } if consumer.ConsumerZnode == nil { this.Ui.Warn(fmt.Sprintf("%+v has no znode", consumer)) continue } lines = append(lines, fmt.Sprintf("%s|%s/%s|%s|%s|%s|%s|%s", group, consumer.Topic, consumer.PartitionId, gofmt.Comma(consumer.ProducerOffset), gofmt.Comma(consumer.ConsumerOffset), gofmt.Comma(consumer.Lag), gofmt.PrettySince(consumer.Mtime.Time()), gofmt.PrettySince(consumer.ConsumerZnode.Uptime()))) } } if len(lines) > 1 { this.Ui.Info(fmt.Sprintf("%s ▾", zkcluster.Name())) this.Ui.Output(columnize.SimpleFormat(lines)) } }
func (c *ConsumerZnode) String() string { return fmt.Sprintf("host:%s, sub:%+v, uptime:%s", c.Host(), c.Subscription, gofmt.PrettySince(c.Uptime())) }
func (c *ControllerMeta) String() string { return fmt.Sprintf("%s epoch:%s/%s %s", c.Broker.Id, c.Epoch, gofmt.PrettySince(c.Mtime.Time()), c.Broker.String()) }
func (b BrokerZnode) String() string { return fmt.Sprintf("%s ver:%d uptime:%s", b.Addr(), b.Version, gofmt.PrettySince(b.Uptime())) }
func (this *Kateway) Run(args []string) (exitCode int) { cmdFlags := flag.NewFlagSet("kateway", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&this.zone, "z", "", "") cmdFlags.BoolVar(&this.configMode, "cf", false, "") cmdFlags.StringVar(&this.id, "id", "", "") cmdFlags.BoolVar(&this.install, "i", false, "") cmdFlags.BoolVar(&this.longFmt, "l", false, "") cmdFlags.StringVar(&this.configOption, "option", "", "") cmdFlags.BoolVar(&this.versionOnly, "ver", false, "") cmdFlags.BoolVar(&this.flameGraph, "flame", false, "") cmdFlags.StringVar(&this.logLevel, "loglevel", "", "") cmdFlags.StringVar(&this.visualLog, "visualog", "", "") cmdFlags.BoolVar(&this.showZkNodes, "zk", false, "") cmdFlags.BoolVar(&this.checkup, "checkup", false, "") cmdFlags.BoolVar(&this.benchmark, "bench", false, "") cmdFlags.StringVar(&this.benchmarkMaster, "master", "", "") cmdFlags.BoolVar(&this.pub, "pub", false, "") cmdFlags.BoolVar(&this.sub, "sub", false, "") cmdFlags.BoolVar(&this.benchmarkAsync, "async", false, "") cmdFlags.BoolVar(&this.curl, "curl", false, "") if err := cmdFlags.Parse(args); err != nil { return 2 } if this.benchmark { if validateArgs(this, this.Ui). require("-z"). requireAdminRights("-z"). invalid(args) { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) zone := ctx.Zone(zkzone.Name()) this.benchApp = zone.SmokeApp this.benchSecret = zone.SmokeSecret this.benchTopic = zone.SmokeTopic this.benchVer = zone.SmokeTopicVersion this.benchPubEndpoint = zone.PubEndpoint if this.id != "" { kws, err := zkzone.KatewayInfos() swallow(err) for _, kw := range kws { if kw.Id == this.id { this.benchPubEndpoint = kw.PubAddr break } } } this.benchId = fmt.Sprintf("%s-%s", ctx.Hostname(), strings.Replace(uuid.New(), "-", "", -1)) this.runBenchmark(zkzone) return } if this.flameGraph { if validateArgs(this, this.Ui). require("-z", "-id"). requireAdminRights("-z"). invalid(args) { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.generateFlameGraph(zkzone) return } if this.visualLog != "" { this.doVisualize() return } if this.pub { zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.runPub(zkzone) return } if this.sub { zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.runSub(zkzone) return } if this.install { if validateArgs(this, this.Ui). require("-z"). invalid(args) { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.installGuide(zkzone) return } if this.configOption != "" { this.configMode = true } if this.configMode { if validateArgs(this, this.Ui). require("-z"). requireAdminRights("-z"). invalid(args) { return 2 } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) if this.logLevel != "" { if this.id != "" { kw := zkzone.KatewayInfoById(this.id) if kw == nil { panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id)) } this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel)) } else { // apply on all kateways kws, _ := zkzone.KatewayInfos() for _, kw := range kws { this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel)) } } } if this.configOption != "" { parts := strings.SplitN(this.configOption, "=", 2) if len(parts) != 2 { this.Ui.Error("usage: key=value") return } k, v := parts[0], parts[1] if this.id != "" { kw := zkzone.KatewayInfoById(this.id) if kw == nil { panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id)) } this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v)) } else { // apply on all kateways kws, _ := zkzone.KatewayInfos() for _, kw := range kws { this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v)) } } } return } if this.checkup { if this.zone == "" { forAllSortedZones(func(zkzone *zk.ZkZone) { this.runCheckup(zkzone) }) return } zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone))) this.runCheckup(zkzone) return } if this.showZkNodes { this.Ui.Output(fmt.Sprintf(`%s pubsub manager db dsn %s job db cluster config %s turn off webhook dir`, color.Green("%-50s", zk.KatewayMysqlPath), color.Green("%-50s", zk.PubsubJobConfig), color.Green("%-50s", zk.PubsubWebhooksOff))) return } // display mode lines := make([]string, 0) header := "Zone|Id|Ip|Pprof|Build|Cpu|Heap|Obj|Go|P/S|hhIn/hhOut|Uptime" lines = append(lines, header) forSortedZones(func(zkzone *zk.ZkZone) { if this.zone != "" && zkzone.Name() != this.zone { return } if !this.versionOnly { mysqlDsn, err := zkzone.KatewayMysqlDsn() if err != nil { this.Ui.Warn(fmt.Sprintf("kateway[%s] mysql DSN not set on zk yet", zkzone.Name())) this.Ui.Output(fmt.Sprintf("e,g. %s -> pubsub:pubsub@tcp(10.77.135.217:10010)/pubsub?charset=utf8&timeout=10s", zk.KatewayMysqlPath)) } else { this.Ui.Output(fmt.Sprintf("zone[%s] manager db: %s", color.Cyan(zkzone.Name()), mysqlDsn)) } } kateways, err := zkzone.KatewayInfos() if err != nil { if err == zklib.ErrNoNode { this.Ui.Output("no kateway running") return } else { swallow(err) } } for _, kw := range kateways { if this.id != "" && this.id != kw.Id { continue } statusMap, _ := this.getKatewayStatusMap(kw.ManAddr) logLevel, _ := statusMap["loglevel"].(string) heapSize, _ := statusMap["heap"].(string) heapObjs, _ := statusMap["objects"].(string) pubConn, _ := statusMap["pubconn"].(string) hhAppendN, _ := statusMap["hh_appends"].(string) hhDeliverN, _ := statusMap["hh_delivers"].(string) subConn, _ := statusMap["subconn"].(string) goN, _ := statusMap["goroutines"].(string) if this.versionOnly { pprofAddr := kw.DebugAddr if len(pprofAddr) > 0 && pprofAddr[0] == ':' { pprofAddr = kw.Ip + pprofAddr } pprofAddr = fmt.Sprintf("%s/debug/pprof/", pprofAddr) lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s/%s|%s|%s|%s|%s|%s/%s|%s/%s|%s", zkzone.Name(), kw.Id, kw.Ip, pprofAddr, kw.Build, kw.BuiltAt, kw.Cpu, heapSize, heapObjs, goN, pubConn, subConn, hhAppendN, hhDeliverN, gofmt.PrettySince(kw.Ctime))) continue } this.Ui.Info(fmt.Sprintf("id:%-2s host:%s cpu:%-2s up:%s", kw.Id, kw.Host, kw.Cpu, gofmt.PrettySince(kw.Ctime))) this.Ui.Output(fmt.Sprintf(" ver: %s\n arch: %s\n build: %s\n built: %s\n log: %s\n pub: %s\n sub: %s\n man: %s\n dbg: %s", kw.Ver, kw.Arch, color.Red(kw.Build), kw.BuiltAt, logLevel, kw.PubAddr, kw.SubAddr, kw.ManAddr, kw.DebugAddr, )) if this.longFmt { this.Ui.Output(" full status:") this.Ui.Output(this.getKatewayStatus(kw.ManAddr)) } } }) if this.versionOnly && len(lines) > 1 { fmt.Println(columnize.SimpleFormat(lines)) } return }
func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) []string { brokerList := zkcluster.BrokerList() if len(brokerList) == 0 { this.Ui.Warn(fmt.Sprintf("%s empty brokers", zkcluster.Name())) return nil } kfk, err := sarama.NewClient(brokerList, saramaConfig()) if err != nil { this.Ui.Error(fmt.Sprintf("%s %+v %s", zkcluster.Name(), brokerList, err.Error())) return nil } defer kfk.Close() topics, err := kfk.Topics() swallow(err) if len(topics) == 0 { return nil } lines := make([]string, 0, 10) for _, topic := range topics { // get partitions and check if some dead alivePartitions, err := kfk.WritablePartitions(topic) if err != nil { this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch writable partitions: %v", zkcluster.Name(), topic, err)) continue } partions, err := kfk.Partitions(topic) if err != nil { this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch partitions: %v", zkcluster.Name(), topic, err)) continue } if len(alivePartitions) != len(partions) { this.Ui.Error(fmt.Sprintf("%s topic[%s] has %s partitions: %+v/%+v", zkcluster.Name(), topic, color.Red("dead"), alivePartitions, partions)) } for _, partitionID := range alivePartitions { replicas, err := kfk.Replicas(topic, partitionID) if err != nil { this.Ui.Error(fmt.Sprintf("%s topic[%s] P:%d: %v", zkcluster.Name(), topic, partitionID, err)) continue } isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID) underReplicated := false if len(isr) != len(replicas) { underReplicated = true } if underReplicated { leader, err := kfk.Leader(topic, partitionID) swallow(err) latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) swallow(err) oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest) swallow(err) lines = append(lines, fmt.Sprintf("\t%s Partition:%d/%s Leader:%d Replicas:%+v Isr:%+v/%s Offset:%d-%d Num:%d", topic, partitionID, gofmt.PrettySince(partitionCtime), leader.ID(), replicas, isr, gofmt.PrettySince(isrMtime), oldestOffset, latestOffset, latestOffset-oldestOffset)) } } } return lines }
func (this *Topology) displayZoneTopology(zkzone *zk.ZkZone) { this.Ui.Output(zkzone.Name()) // {cluster: {topic: brokerHostInfo}} brokerInstances := make(map[string]map[string]*brokerHostInfo) zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) { if len(liveBrokers) == 0 { this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster)) return } if this.cluster != "" && this.cluster != cluster { return } brokerInstances[cluster] = make(map[string]*brokerHostInfo) for _, broker := range liveBrokers { if !patternMatched(broker.Host, this.hostPattern) { continue } if _, present := brokerInstances[cluster][broker.Host]; !present { brokerInstances[cluster][broker.Host] = newBrokerHostInfo() } brokerInstances[cluster][broker.Host].addPort(broker.Port, broker.Uptime()) } // find how many partitions a broker is leading zkcluster := zkzone.NewCluster(cluster) brokerList := zkcluster.BrokerList() if len(brokerList) == 0 { this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster)) return } kfk, err := sarama.NewClient(brokerList, sarama.NewConfig()) if err != nil { this.Ui.Error(color.Red(" %+v %s", brokerList, err.Error())) return } topics, err := kfk.Topics() swallow(err) for _, topic := range topics { partions, err := kfk.WritablePartitions(topic) swallow(err) for _, partitionID := range partions { leader, err := kfk.Leader(topic, partitionID) swallow(err) host, _, err := net.SplitHostPort(leader.Addr()) swallow(err) if !patternMatched(host, this.hostPattern) { continue } latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) if err != nil { this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err)) continue } oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest) if err != nil { this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err)) continue } brokerInstances[cluster][host].topicMsgs[topic] += (latestOffset - oldestOffset) brokerInstances[cluster][host].addTopicPartition(topic, partitionID) } } }) hosts := make(map[string]struct{}) zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { for host, _ := range brokerInstances[zkcluster.Name()] { hosts[host] = struct{}{} } }) sortedHosts := make([]string, 0) for host, _ := range hosts { sortedHosts = append(sortedHosts, host) } sort.Strings(sortedHosts) // sort by host ip sortedClusters := make([]string, 0, len(brokerInstances)) for c, _ := range brokerInstances { sortedClusters = append(sortedClusters, c) } sort.Strings(sortedClusters) portN := 0 hostN := 0 topicN := 0 partitionN := 0 for _, host := range sortedHosts { tn := 0 pn := 0 mn := int64(0) ports := make([]int, 0) for _, cluster := range sortedClusters { if _, present := brokerInstances[cluster][host]; !present { continue } tn += len(brokerInstances[cluster][host].topicPartitions) pn += brokerInstances[cluster][host].leadingPartitions() mn += brokerInstances[cluster][host].totalMsgsInStock() ports = append(ports, brokerInstances[cluster][host].tcpPorts...) } portN += len(ports) topicN += tn partitionN += pn hostN += 1 this.Ui.Output(fmt.Sprintf(" %s leading: %2dT %3dP %15sM ports %2d:%+v", color.Green("%15s", host), tn, pn, gofmt.Comma(mn), len(ports), ports)) if this.verbose { for _, cluster := range sortedClusters { if _, present := brokerInstances[cluster][host]; !present { continue } for _, tcpPort := range brokerInstances[cluster][host].tcpPorts { this.Ui.Output(fmt.Sprintf("%40d %s", tcpPort, gofmt.PrettySince(brokerInstances[cluster][host].uptimes[tcpPort]))) } } for _, cluster := range sortedClusters { if _, present := brokerInstances[cluster][host]; !present { continue } this.Ui.Output(color.Magenta("%30s", cluster)) for topic, partitions := range brokerInstances[cluster][host].topicPartitions { this.Ui.Output(fmt.Sprintf("%40s: %15sM P%2d %+v", topic, gofmt.Comma(brokerInstances[cluster][host].topicMsgs[topic]), len(partitions), partitions)) } } } } this.Ui.Output(fmt.Sprintf("%17s host:%d, topic:%d, partition:%d, instance:%d", "-TOTAL-", hostN, topicN, partitionN, portN)) }
func (this *Topics) Run(args []string) (exitCode int) { var ( zone string cluster string addTopic string delTopic string killTopic, restoreTopic string replicas int partitions int retentionInMinute int resetConf bool debug bool summaryMode bool configged bool ) cmdFlags := flag.NewFlagSet("brokers", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "") cmdFlags.StringVar(&this.topicPattern, "t", "", "") cmdFlags.StringVar(&cluster, "c", "", "") cmdFlags.BoolVar(&this.verbose, "l", false, "") cmdFlags.BoolVar(&this.ipInNumber, "n", false, "") cmdFlags.StringVar(&addTopic, "add", "", "") cmdFlags.BoolVar(&summaryMode, "sum", false, "") cmdFlags.StringVar(&killTopic, "kill", "", "") cmdFlags.StringVar(&restoreTopic, "restore", "", "") cmdFlags.BoolVar(&this.plainMode, "plain", false, "") cmdFlags.StringVar(&delTopic, "del", "", "") cmdFlags.IntVar(&partitions, "partitions", 1, "") cmdFlags.DurationVar(&this.since, "since", 0, "") cmdFlags.BoolVar(&configged, "cf", false, "") cmdFlags.BoolVar(&debug, "debug", false, "") cmdFlags.BoolVar(&resetConf, "cfreset", false, "") cmdFlags.Int64Var(&this.count, "count", 0, "") cmdFlags.IntVar(&retentionInMinute, "retention", -1, "") cmdFlags.IntVar(&replicas, "replicas", 2, "") if err := cmdFlags.Parse(args); err != nil { return 1 } if validateArgs(this, this.Ui). on("-add", "-c"). on("-del", "-c"). on("-retention", "-c", "-t"). on("-cfreset", "-c", "-t"). requireAdminRights("-add", "-del", "-retention"). invalid(args) { return 2 } if this.count > 0 { this.verbose = true } if debug { sarama.Logger = log.New(os.Stderr, color.Magenta("[sarama]"), log.LstdFlags) } if addTopic != "" { zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) zkcluster := zkzone.NewCluster(cluster) swallow(this.addTopic(zkcluster, addTopic, replicas, partitions)) return } else if delTopic != "" { zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) zkcluster := zkzone.NewCluster(cluster) swallow(this.delTopic(zkcluster, delTopic)) return } ensureZoneValid(zone) zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) if retentionInMinute > 0 { zkcluster := zkzone.NewCluster(cluster) this.configTopic(zkcluster, this.topicPattern, retentionInMinute) return } if resetConf { zkcluster := zkzone.NewCluster(cluster) this.resetTopicConfig(zkcluster, this.topicPattern) configged = true // after reset, display most recent znode info } if configged { // output header this.Ui.Output(fmt.Sprintf("%25s %-40s %15s", "cluster", "topic", "mtime")) this.Ui.Output(fmt.Sprintf("%25s %40s %15s", strings.Repeat("-", 25), strings.Repeat("-", 40), strings.Repeat("-", 15))) displayTopicConfigs := func(zkcluster *zk.ZkCluster) { // sort by topic name configs := zkcluster.ConfiggedTopics() sortedTopics := make([]string, 0, len(configs)) for topic, _ := range configs { sortedTopics = append(sortedTopics, topic) } sort.Strings(sortedTopics) for _, topic := range sortedTopics { if !patternMatched(topic, this.topicPattern) { continue } configInfo := configs[topic] this.Ui.Output(fmt.Sprintf("%25s %40s %15s %s", zkcluster.Name(), topic, gofmt.PrettySince(configInfo.Mtime), configInfo.Config)) } } if cluster == "" { zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { displayTopicConfigs(zkcluster) }) } else { zkcluster := zkzone.NewCluster(cluster) displayTopicConfigs(zkcluster) } return } if summaryMode { this.printSummary(zkzone, cluster) return } if !this.verbose { // output header this.Ui.Output(fmt.Sprintf("%30s %-50s", "cluster", "topic")) this.Ui.Output(fmt.Sprintf("%30s %50s", strings.Repeat("-", 30), strings.Repeat("-", 50))) } if cluster != "" { zkcluster := zkzone.NewCluster(cluster) this.displayTopicsOfCluster(zkcluster) this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Topics-", this.topicN)) this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Partitions-", this.partitionN)) if this.verbose { if this.plainMode { fmt.Printf("%25s %s\n", "-FLAT Messages-", gofmt.Comma(this.totalMsgs)) fmt.Printf("%25s %s\n", "-CUM Messages-", gofmt.Comma(this.totalOffsets)) } else { this.Ui.Output(fmt.Sprintf("%25s %s", "-FLAT Messages-", gofmt.Comma(this.totalMsgs))) this.Ui.Output(fmt.Sprintf("%25s %s", "-CUM Messages-", gofmt.Comma(this.totalOffsets))) } } return } // all clusters zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { this.displayTopicsOfCluster(zkcluster) }) this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Topics-", this.topicN)) this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Partitions-", this.partitionN)) if this.verbose { if this.plainMode { fmt.Printf("%25s %s\n", "-FLAT Messages-", gofmt.Comma(this.totalMsgs)) fmt.Printf("%25s %s\n", "-CUM Messages-", gofmt.Comma(this.totalOffsets)) } else { this.Ui.Output(fmt.Sprintf("%25s %s", "-FLAT Messages-", gofmt.Comma(this.totalMsgs))) this.Ui.Output(fmt.Sprintf("%25s %s", "-CUM Messages-", gofmt.Comma(this.totalOffsets))) } } return }
func (this *Topics) displayTopicsOfCluster(zkcluster *zk.ZkCluster) { echoBuffer := func(lines []string) { for _, l := range lines { this.Ui.Output(l) } } linesInTopicMode := make([]string, 0) if this.verbose { linesInTopicMode = this.echoOrBuffer(zkcluster.Name(), linesInTopicMode) } // get all alive brokers within this cluster brokers := zkcluster.Brokers() if len(brokers) == 0 { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s", " ", color.Red("%s empty brokers", zkcluster.Name())), linesInTopicMode) echoBuffer(linesInTopicMode) return } if this.verbose { sortedBrokerIds := make([]string, 0, len(brokers)) for brokerId, _ := range brokers { sortedBrokerIds = append(sortedBrokerIds, brokerId) } sort.Strings(sortedBrokerIds) for _, brokerId := range sortedBrokerIds { if this.ipInNumber { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ", color.Green(brokerId), brokers[brokerId]), linesInTopicMode) } else { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ", color.Green(brokerId), brokers[brokerId].NamedString()), linesInTopicMode) } } } kfk, err := sarama.NewClient(zkcluster.BrokerList(), saramaConfig()) if err != nil { if this.verbose { linesInTopicMode = this.echoOrBuffer(color.Yellow("%5s%+v %s", " ", zkcluster.BrokerList(), err.Error()), linesInTopicMode) } return } defer kfk.Close() topics, err := kfk.Topics() swallow(err) if len(topics) == 0 { if this.topicPattern == "" && this.verbose { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%5s%s", " ", color.Magenta("no topics")), linesInTopicMode) echoBuffer(linesInTopicMode) } return } sortedTopics := make([]string, 0, len(topics)) for _, t := range topics { sortedTopics = append(sortedTopics, t) } sort.Strings(sortedTopics) topicsCtime := zkcluster.TopicsCtime() hasTopicMatched := false for _, topic := range sortedTopics { if !patternMatched(topic, this.topicPattern) { continue } if this.since > 0 && time.Since(topicsCtime[topic]) > this.since { continue } this.topicN++ hasTopicMatched = true if this.verbose { linesInTopicMode = this.echoOrBuffer(strings.Repeat(" ", 4)+color.Cyan(topic), linesInTopicMode) } // get partitions and check if some dead alivePartitions, err := kfk.WritablePartitions(topic) swallow(err) partions, err := kfk.Partitions(topic) swallow(err) if len(alivePartitions) != len(partions) { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %s P: %s/%+v", zkcluster.Name(), color.Cyan("%-50s", topic), color.Red("partial dead"), color.Green("%+v", alivePartitions), partions), linesInTopicMode) } replicas, err := kfk.Replicas(topic, partions[0]) if err != nil { this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partions[0], err)) } this.partitionN += len(partions) if !this.verbose { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %3dP %dR %s", zkcluster.Name(), color.Cyan("%-50s", topic), len(partions), len(replicas), gofmt.PrettySince(topicsCtime[topic])), linesInTopicMode) continue } for _, partitionID := range alivePartitions { leader, err := kfk.Leader(topic, partitionID) swallow(err) replicas, err := kfk.Replicas(topic, partitionID) if err != nil { this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partitionID, err)) } isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID) isrMtimeSince := gofmt.PrettySince(isrMtime) if time.Since(isrMtime).Hours() < 24 { // ever out of sync last 24h isrMtimeSince = color.Magenta(isrMtimeSince) } underReplicated := false if len(isr) != len(replicas) { underReplicated = true } latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) swallow(err) oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest) swallow(err) if this.count > 0 && (latestOffset-oldestOffset) < this.count { continue } this.totalMsgs += latestOffset - oldestOffset this.totalOffsets += latestOffset if !underReplicated { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%+v Offset:%16s - %-16s Num:%-15s %s-%s", partitionID, color.Green("%d", leader.ID()), replicas, isr, gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset), gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode) } else { // use red for alert linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%s Offset:%16s - %-16s Num:%-15s %s-%s", partitionID, color.Green("%d", leader.ID()), replicas, color.Red("%+v", isr), gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset), gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode) } } } if this.topicPattern != "" { if hasTopicMatched { echoBuffer(linesInTopicMode) } } else { echoBuffer(linesInTopicMode) } }
func (this *Consumers) printConsumersByGroupTable(zkzone *zk.ZkZone, clusterPattern string) { lines := make([]string, 0) header := "Zone|Cluster|M|Host|ConsumerGroup|Topic/Partition|Offset|Uptime" lines = append(lines, header) zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { groupTopicsMap := make(map[string]map[string]struct{}) // group:sub topics if !patternMatched(zkcluster.Name(), clusterPattern) { return } consumerGroups := zkcluster.ConsumerGroups() sortedGroups := make([]string, 0, len(consumerGroups)) for group, _ := range consumerGroups { if !patternMatched(group, this.groupPattern) { continue } sortedGroups = append(sortedGroups, group) } sort.Strings(sortedGroups) for _, group := range sortedGroups { consumers := consumerGroups[group] if this.onlineOnly && len(consumers) == 0 { continue } if len(consumers) > 0 { // sort by host sortedIds := make([]string, 0) consumersMap := make(map[string]*zk.ConsumerZnode) for _, c := range consumers { sortedIds = append(sortedIds, c.Id) consumersMap[c.Id] = c } sort.Strings(sortedIds) for _, consumerId := range sortedIds { c := consumersMap[consumerId] for topic, _ := range c.Subscription { if !patternMatched(topic, this.topicPattern) { continue } if len(groupTopicsMap[group]) == 0 { groupTopicsMap[group] = make(map[string]struct{}, 5) } groupTopicsMap[group][topic] = struct{}{} ownerByPartition := zkcluster.OwnersOfGroupByTopic(group, topic) partitionsWithOffset := make(map[string]struct{}) for _, offset := range this.displayGroupOffsets(zkcluster, group, topic, false) { onlineSymbol := "◉" isOwner := false if ownerByPartition[offset.partitionId] == consumerId { onlineSymbol += "*" // owned by this consumer isOwner = true } if this.ownerOnly && !isOwner { continue } partitionsWithOffset[offset.partitionId] = struct{}{} lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s|%s", zkzone.Name(), zkcluster.Name(), onlineSymbol, c.Host(), group+"@"+c.Id[len(c.Id)-12:], fmt.Sprintf("%s/%s", offset.topic, offset.partitionId), offset.offset, gofmt.PrettySince(c.Uptime()))) } for partitionId, _ := range ownerByPartition { if _, present := partitionsWithOffset[partitionId]; !present { // this consumer is owner online, but has no offset onlineSymbol := "◉" isOwner := false if ownerByPartition[partitionId] == consumerId { onlineSymbol += "*" isOwner = true } if this.ownerOnly && !isOwner { continue } lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s|%s|?|%s", zkzone.Name(), zkcluster.Name(), onlineSymbol, c.Host(), group+"@"+c.Id[len(c.Id)-12:], fmt.Sprintf("%s/%s", topic, partitionId), gofmt.PrettySince(c.Uptime()))) } } } } } else { // offline for _, offset := range this.displayGroupOffsets(zkcluster, group, "", false) { if !patternMatched(offset.topic, this.topicPattern) { continue } lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s|%s", zkzone.Name(), zkcluster.Name(), "◎", " ", group, fmt.Sprintf("%s/%s", offset.topic, offset.partitionId), offset.offset, " ")) } } } for group, topics := range groupTopicsMap { if len(topics) > 1 { // the same consumer group is consuming more than 1 topics topicsLabel := make([]string, 0, len(topics)) for t, _ := range topics { topicsLabel = append(topicsLabel, t) } this.Ui.Warn(fmt.Sprintf("%35s consuming: %+v", group, topicsLabel)) } } }) if !this.warnOnly { this.Ui.Output(columnize.SimpleFormat(lines)) } }