Exemple #1
0
func (this *Stat) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("stat", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.zone == "" {
		this.Ui.Error("unknown zone")
		return 2
	}

	if len(args) == 0 {
		this.Ui.Error("missing path")
		return 2
	}

	this.path = args[len(args)-1]

	zkzone := gzk.NewZkZone(gzk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	defer zkzone.Close()
	conn := zkzone.Conn()

	_, stat, err := conn.Get(this.path)
	must(err)
	this.Ui.Output(fmt.Sprintf("%# v", pretty.Formatter(*stat)))
	ctime := gzk.ZkTimestamp(stat.Ctime).Time()
	mtime := gzk.ZkTimestamp(stat.Mtime).Time()
	this.Ui.Output(fmt.Sprintf("ctime: %s, mtime: %s",
		gofmt.PrettySince(ctime), gofmt.PrettySince(mtime)))
	return
}
Exemple #2
0
// Print all controllers of all clusters within a zone.
func (this *Controllers) printControllers(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedControllers(func(cluster string, controller *zk.ControllerMeta) {
		if !patternMatched(cluster, this.cluster) {
			return
		}

		this.Ui.Output(strings.Repeat(" ", 4) + cluster)
		if controller == nil {
			this.Ui.Output(fmt.Sprintf("\t%s", color.Red("empty")))
		} else {
			epochSince := time.Since(controller.Mtime.Time())
			epochSinceStr := gofmt.PrettySince(controller.Mtime.Time())
			if epochSince < time.Hour*2*24 {
				epochSinceStr = color.Red(epochSinceStr)
			}
			this.Ui.Output(fmt.Sprintf("\t%-2s %21s epoch:%2s/%-20s uptime:%s",
				controller.Broker.Id, controller.Broker.Addr(),
				controller.Epoch,
				epochSinceStr,
				gofmt.PrettySince(controller.Broker.Uptime())))
		}
	})

}
Exemple #3
0
func (this *Brokers) clusterBrokers(zone, cluster string, brokers map[string]*zk.BrokerZnode) []string {
	if !patternMatched(cluster, this.cluster) {
		return nil
	}

	if brokers == nil || len(brokers) == 0 {
		return []string{fmt.Sprintf("%s|%s|%s|%s|%s",
			zone, cluster, " ", color.Red("empty brokers"), " ")}
	}

	lines := make([]string, 0, len(brokers))
	if this.staleOnly {
		// try each broker's aliveness
		for brokerId, broker := range brokers {
			cf := sarama.NewConfig()
			cf.Net.ReadTimeout = time.Second * 4
			cf.Net.WriteTimeout = time.Second * 4
			kfk, err := sarama.NewClient([]string{broker.Addr()}, cf)
			if err != nil {
				lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
					zone, cluster,
					brokerId, broker.Addr(),
					fmt.Sprintf("%s: %v", gofmt.PrettySince(broker.Uptime()), err)))
			} else {
				kfk.Close()
			}
		}

		return lines
	}

	// sort by broker id
	sortedBrokerIds := make([]string, 0, len(brokers))
	for brokerId, _ := range brokers {
		sortedBrokerIds = append(sortedBrokerIds, brokerId)
	}
	sort.Strings(sortedBrokerIds)

	for _, brokerId := range sortedBrokerIds {
		b := brokers[brokerId]
		uptime := gofmt.PrettySince(b.Uptime())
		if time.Since(b.Uptime()) < time.Hour*24*7 {
			uptime = color.Green(uptime)
		}
		if this.ipInNumber {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
				zone, cluster,
				brokerId, b.Addr(),
				gofmt.PrettySince(b.Uptime())))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
				zone, cluster,
				brokerId, b.NamedAddr(),
				gofmt.PrettySince(b.Uptime())))
		}

	}
	return lines
}
Exemple #4
0
func (b BrokerZnode) NamedString() string {
	addr, _ := b.NamedAddr()
	return fmt.Sprintf("%s ver:%d uptime:%s",
		addr,
		b.Version,
		gofmt.PrettySince(b.Uptime()))
}
Exemple #5
0
func (this *Kguard) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("kguard", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&this.longFmt, "l", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	kguards, err := zkzone.KguardInfos()
	if err != nil {
		this.Ui.Error(fmt.Sprintf("%s %v", zk.KguardLeaderPath, err.Error()))
		return
	}

	leader := kguards[0]
	this.Ui.Output(fmt.Sprintf("%s(out of %d candidates) up: %s",
		color.Green(leader.Host), leader.Candidates,
		gofmt.PrettySince(leader.Ctime)))

	if this.longFmt {
		this.showKguardVersion(leader.Host)
		this.showStats(leader.Host)
	}

	return
}
Exemple #6
0
func (this *Lags) printConsumersLag(zkcluster *zk.ZkCluster) {
	// sort by group name
	consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern)
	sortedGroups := make([]string, 0, len(consumersByGroup))
	for group, _ := range consumersByGroup {
		sortedGroups = append(sortedGroups, group)
	}
	sort.Strings(sortedGroups)

	for _, group := range sortedGroups {
		lines := make([]string, 0, 100)

		sortedTopicAndPartitionIds := make([]string, 0)
		consumers := make(map[string]zk.ConsumerMeta)
		for _, t := range consumersByGroup[group] {
			key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId)
			sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key)

			consumers[key] = t
		}
		sort.Strings(sortedTopicAndPartitionIds)

		for _, topicAndPartitionId := range sortedTopicAndPartitionIds {
			consumer := consumers[topicAndPartitionId]

			if !patternMatched(consumer.Topic, this.topicPattern) {
				continue
			}

			var (
				lagOutput string
				symbol    string
			)
			if consumer.Lag > int64(this.lagThreshold) {
				lagOutput = color.Red("%15s", gofmt.Comma(consumer.Lag))
				if consumer.Online {
					symbol = color.Yellow("⚠︎︎")
				} else {
					symbol = color.Yellow("◎")
				}
			} else {
				lagOutput = color.Blue("%15s", gofmt.Comma(consumer.Lag))
				if consumer.Online {
					symbol = color.Green("◉")
				} else {
					symbol = color.Yellow("◎")
				}
			}

			if consumer.Online {
				if this.problematicMode && consumer.Lag <= int64(this.lagThreshold) {
					continue
				}

				var (
					host   string
					uptime string
				)
				if consumer.ConsumerZnode == nil {
					host = "unrecognized"
					uptime = "-"
				} else {
					host = color.Green("%s", consumer.ConsumerZnode.Host())
					if time.Since(consumer.ConsumerZnode.Uptime()) < time.Hour {
						uptime = color.Magenta(gofmt.PrettySince(consumer.ConsumerZnode.Uptime()))
					} else {
						uptime = gofmt.PrettySince(consumer.ConsumerZnode.Uptime())
					}
				}

				lines = append(lines, fmt.Sprintf("\t%s %35s/%-2s %12s -> %-15s %s %-10s %s %s",
					symbol,
					consumer.Topic, consumer.PartitionId,
					gofmt.Comma(consumer.ProducerOffset),
					gofmt.Comma(consumer.ConsumerOffset),
					lagOutput,
					gofmt.PrettySince(consumer.Mtime.Time()),
					host, uptime))
			} else if !this.onlineOnly {
				lines = append(lines, fmt.Sprintf("\t%s %35s/%-2s %12s -> %-12s %s %s",
					symbol,
					consumer.Topic, consumer.PartitionId,
					gofmt.Comma(consumer.ProducerOffset),
					gofmt.Comma(consumer.ConsumerOffset),
					lagOutput,
					gofmt.PrettySince(consumer.Mtime.Time())))
			}
		}

		if len(lines) > 0 {
			this.Ui.Output(strings.Repeat(" ", 4) + group)
			for _, l := range lines {
				this.Ui.Output(l)
			}
		}
	}
}
Exemple #7
0
func (this *Lags) printConsumersLagTable(zkcluster *zk.ZkCluster) {
	lines := make([]string, 0)
	header := "ConsumerGroup|Topic/Partition|Produced|Consumed|Lag|Committed|Uptime"
	lines = append(lines, header)

	// sort by group name
	consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern)
	sortedGroups := make([]string, 0, len(consumersByGroup))
	for group, _ := range consumersByGroup {
		sortedGroups = append(sortedGroups, group)
	}
	sort.Strings(sortedGroups)

	for _, group := range sortedGroups {
		if !patternMatched(group, this.groupPattern) {
			continue
		}

		sortedTopicAndPartitionIds := make([]string, 0, len(consumersByGroup[group]))
		consumers := make(map[string]zk.ConsumerMeta)
		for _, t := range consumersByGroup[group] {
			key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId)
			sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key)

			consumers[key] = t
		}
		sort.Strings(sortedTopicAndPartitionIds)

		for _, topicAndPartitionId := range sortedTopicAndPartitionIds {
			consumer := consumers[topicAndPartitionId]

			if !patternMatched(consumer.Topic, this.topicPattern) {
				continue
			}
			if !consumer.Online {
				continue
			}
			if this.problematicMode && consumer.Lag <= int64(this.lagThreshold) {
				continue
			}
			if consumer.ConsumerZnode == nil {
				this.Ui.Warn(fmt.Sprintf("%+v has no znode", consumer))
				continue
			}

			lines = append(lines,
				fmt.Sprintf("%s|%s/%s|%s|%s|%s|%s|%s",
					group,
					consumer.Topic, consumer.PartitionId,
					gofmt.Comma(consumer.ProducerOffset),
					gofmt.Comma(consumer.ConsumerOffset),
					gofmt.Comma(consumer.Lag),
					gofmt.PrettySince(consumer.Mtime.Time()),
					gofmt.PrettySince(consumer.ConsumerZnode.Uptime())))
		}
	}

	if len(lines) > 1 {
		this.Ui.Info(fmt.Sprintf("%s ▾", zkcluster.Name()))
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}
Exemple #8
0
func (c *ConsumerZnode) String() string {
	return fmt.Sprintf("host:%s, sub:%+v, uptime:%s",
		c.Host(), c.Subscription, gofmt.PrettySince(c.Uptime()))
}
Exemple #9
0
func (c *ControllerMeta) String() string {
	return fmt.Sprintf("%s epoch:%s/%s %s", c.Broker.Id, c.Epoch,
		gofmt.PrettySince(c.Mtime.Time()), c.Broker.String())
}
Exemple #10
0
func (b BrokerZnode) String() string {
	return fmt.Sprintf("%s ver:%d uptime:%s",
		b.Addr(),
		b.Version,
		gofmt.PrettySince(b.Uptime()))
}
Exemple #11
0
func (this *Kateway) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("kateway", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.BoolVar(&this.configMode, "cf", false, "")
	cmdFlags.StringVar(&this.id, "id", "", "")
	cmdFlags.BoolVar(&this.install, "i", false, "")
	cmdFlags.BoolVar(&this.longFmt, "l", false, "")
	cmdFlags.StringVar(&this.configOption, "option", "", "")
	cmdFlags.BoolVar(&this.versionOnly, "ver", false, "")
	cmdFlags.BoolVar(&this.flameGraph, "flame", false, "")
	cmdFlags.StringVar(&this.logLevel, "loglevel", "", "")
	cmdFlags.StringVar(&this.visualLog, "visualog", "", "")
	cmdFlags.BoolVar(&this.showZkNodes, "zk", false, "")
	cmdFlags.BoolVar(&this.checkup, "checkup", false, "")
	cmdFlags.BoolVar(&this.benchmark, "bench", false, "")
	cmdFlags.StringVar(&this.benchmarkMaster, "master", "", "")
	cmdFlags.BoolVar(&this.pub, "pub", false, "")
	cmdFlags.BoolVar(&this.sub, "sub", false, "")
	cmdFlags.BoolVar(&this.benchmarkAsync, "async", false, "")
	cmdFlags.BoolVar(&this.curl, "curl", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 2
	}

	if this.benchmark {
		if validateArgs(this, this.Ui).
			require("-z").
			requireAdminRights("-z").
			invalid(args) {
			return 2
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		zone := ctx.Zone(zkzone.Name())
		this.benchApp = zone.SmokeApp
		this.benchSecret = zone.SmokeSecret
		this.benchTopic = zone.SmokeTopic
		this.benchVer = zone.SmokeTopicVersion
		this.benchPubEndpoint = zone.PubEndpoint
		if this.id != "" {
			kws, err := zkzone.KatewayInfos()
			swallow(err)
			for _, kw := range kws {
				if kw.Id == this.id {
					this.benchPubEndpoint = kw.PubAddr
					break
				}
			}
		}
		this.benchId = fmt.Sprintf("%s-%s", ctx.Hostname(), strings.Replace(uuid.New(), "-", "", -1))
		this.runBenchmark(zkzone)
		return
	}

	if this.flameGraph {
		if validateArgs(this, this.Ui).
			require("-z", "-id").
			requireAdminRights("-z").
			invalid(args) {
			return 2
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.generateFlameGraph(zkzone)
		return
	}

	if this.visualLog != "" {
		this.doVisualize()
		return
	}

	if this.pub {
		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.runPub(zkzone)
		return
	}

	if this.sub {
		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.runSub(zkzone)
		return
	}

	if this.install {
		if validateArgs(this, this.Ui).
			require("-z").
			invalid(args) {
			return 2
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.installGuide(zkzone)
		return
	}

	if this.configOption != "" {
		this.configMode = true
	}

	if this.configMode {
		if validateArgs(this, this.Ui).
			require("-z").
			requireAdminRights("-z").
			invalid(args) {
			return 2
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		if this.logLevel != "" {
			if this.id != "" {
				kw := zkzone.KatewayInfoById(this.id)
				if kw == nil {
					panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id))
				}

				this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel))
			} else {
				// apply on all kateways
				kws, _ := zkzone.KatewayInfos()
				for _, kw := range kws {
					this.callKateway(kw, "PUT", fmt.Sprintf("v1/log/%s", this.logLevel))
				}
			}
		}

		if this.configOption != "" {
			parts := strings.SplitN(this.configOption, "=", 2)
			if len(parts) != 2 {
				this.Ui.Error("usage: key=value")
				return
			}
			k, v := parts[0], parts[1]
			if this.id != "" {
				kw := zkzone.KatewayInfoById(this.id)
				if kw == nil {
					panic(fmt.Sprintf("kateway %s invalid entry found in zk", this.id))
				}

				this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v))
			} else {
				// apply on all kateways
				kws, _ := zkzone.KatewayInfos()
				for _, kw := range kws {
					this.callKateway(kw, "PUT", fmt.Sprintf("v1/options/%s/%s", k, v))
				}
			}
		}

		return
	}

	if this.checkup {
		if this.zone == "" {
			forAllSortedZones(func(zkzone *zk.ZkZone) {
				this.runCheckup(zkzone)
			})

			return
		}

		zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
		this.runCheckup(zkzone)
		return
	}

	if this.showZkNodes {
		this.Ui.Output(fmt.Sprintf(`%s pubsub manager db dsn
%s job db cluster config
%s turn off webhook dir`,
			color.Green("%-50s", zk.KatewayMysqlPath),
			color.Green("%-50s", zk.PubsubJobConfig),
			color.Green("%-50s", zk.PubsubWebhooksOff)))
		return
	}

	// display mode
	lines := make([]string, 0)
	header := "Zone|Id|Ip|Pprof|Build|Cpu|Heap|Obj|Go|P/S|hhIn/hhOut|Uptime"
	lines = append(lines, header)
	forSortedZones(func(zkzone *zk.ZkZone) {
		if this.zone != "" && zkzone.Name() != this.zone {
			return
		}

		if !this.versionOnly {
			mysqlDsn, err := zkzone.KatewayMysqlDsn()
			if err != nil {
				this.Ui.Warn(fmt.Sprintf("kateway[%s] mysql DSN not set on zk yet", zkzone.Name()))
				this.Ui.Output(fmt.Sprintf("e,g. %s -> pubsub:pubsub@tcp(10.77.135.217:10010)/pubsub?charset=utf8&timeout=10s",
					zk.KatewayMysqlPath))
			} else {
				this.Ui.Output(fmt.Sprintf("zone[%s] manager db: %s",
					color.Cyan(zkzone.Name()), mysqlDsn))
			}
		}

		kateways, err := zkzone.KatewayInfos()
		if err != nil {
			if err == zklib.ErrNoNode {
				this.Ui.Output("no kateway running")
				return
			} else {
				swallow(err)
			}

		}

		for _, kw := range kateways {
			if this.id != "" && this.id != kw.Id {
				continue
			}

			statusMap, _ := this.getKatewayStatusMap(kw.ManAddr)
			logLevel, _ := statusMap["loglevel"].(string)
			heapSize, _ := statusMap["heap"].(string)
			heapObjs, _ := statusMap["objects"].(string)
			pubConn, _ := statusMap["pubconn"].(string)
			hhAppendN, _ := statusMap["hh_appends"].(string)
			hhDeliverN, _ := statusMap["hh_delivers"].(string)
			subConn, _ := statusMap["subconn"].(string)
			goN, _ := statusMap["goroutines"].(string)

			if this.versionOnly {
				pprofAddr := kw.DebugAddr
				if len(pprofAddr) > 0 && pprofAddr[0] == ':' {
					pprofAddr = kw.Ip + pprofAddr
				}
				pprofAddr = fmt.Sprintf("%s/debug/pprof/", pprofAddr)
				lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s/%s|%s|%s|%s|%s|%s/%s|%s/%s|%s",
					zkzone.Name(),
					kw.Id, kw.Ip,
					pprofAddr, kw.Build, kw.BuiltAt,
					kw.Cpu,
					heapSize, heapObjs,
					goN,
					pubConn, subConn, hhAppendN, hhDeliverN,
					gofmt.PrettySince(kw.Ctime)))
				continue
			}

			this.Ui.Info(fmt.Sprintf("id:%-2s host:%s cpu:%-2s up:%s",
				kw.Id, kw.Host, kw.Cpu,
				gofmt.PrettySince(kw.Ctime)))
			this.Ui.Output(fmt.Sprintf("    ver: %s\n   arch: %s\n  build: %s\n  built: %s\n    log: %s\n    pub: %s\n    sub: %s\n    man: %s\n    dbg: %s",
				kw.Ver,
				kw.Arch,
				color.Red(kw.Build),
				kw.BuiltAt,
				logLevel,
				kw.PubAddr,
				kw.SubAddr,
				kw.ManAddr,
				kw.DebugAddr,
			))

			if this.longFmt {
				this.Ui.Output("    full status:")
				this.Ui.Output(this.getKatewayStatus(kw.ManAddr))
			}

		}

	})

	if this.versionOnly && len(lines) > 1 {
		fmt.Println(columnize.SimpleFormat(lines))
	}

	return
}
Exemple #12
0
func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) []string {
	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		this.Ui.Warn(fmt.Sprintf("%s empty brokers", zkcluster.Name()))
		return nil
	}

	kfk, err := sarama.NewClient(brokerList, saramaConfig())
	if err != nil {
		this.Ui.Error(fmt.Sprintf("%s %+v %s", zkcluster.Name(), brokerList, err.Error()))

		return nil
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		return nil
	}

	lines := make([]string, 0, 10)
	for _, topic := range topics {
		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch writable partitions: %v", zkcluster.Name(), topic, err))
			continue
		}
		partions, err := kfk.Partitions(topic)
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] cannot fetch partitions: %v", zkcluster.Name(), topic, err))
			continue
		}
		if len(alivePartitions) != len(partions) {
			this.Ui.Error(fmt.Sprintf("%s topic[%s] has %s partitions: %+v/%+v", zkcluster.Name(),
				topic, color.Red("dead"), alivePartitions, partions))
		}

		for _, partitionID := range alivePartitions {
			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("%s topic[%s] P:%d: %v", zkcluster.Name(), topic, partitionID, err))
				continue
			}

			isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			if underReplicated {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				swallow(err)

				lines = append(lines, fmt.Sprintf("\t%s Partition:%d/%s Leader:%d Replicas:%+v Isr:%+v/%s Offset:%d-%d Num:%d",
					topic, partitionID,
					gofmt.PrettySince(partitionCtime),
					leader.ID(), replicas, isr,
					gofmt.PrettySince(isrMtime),
					oldestOffset, latestOffset, latestOffset-oldestOffset))
			}
		}
	}

	return lines
}
Exemple #13
0
func (this *Topology) displayZoneTopology(zkzone *zk.ZkZone) {
	this.Ui.Output(zkzone.Name())

	// {cluster: {topic: brokerHostInfo}}
	brokerInstances := make(map[string]map[string]*brokerHostInfo)

	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		if len(liveBrokers) == 0 {
			this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster))
			return
		}
		if this.cluster != "" && this.cluster != cluster {
			return
		}

		brokerInstances[cluster] = make(map[string]*brokerHostInfo)

		for _, broker := range liveBrokers {
			if !patternMatched(broker.Host, this.hostPattern) {
				continue
			}

			if _, present := brokerInstances[cluster][broker.Host]; !present {
				brokerInstances[cluster][broker.Host] = newBrokerHostInfo()
			}
			brokerInstances[cluster][broker.Host].addPort(broker.Port, broker.Uptime())
		}

		// find how many partitions a broker is leading
		zkcluster := zkzone.NewCluster(cluster)
		brokerList := zkcluster.BrokerList()
		if len(brokerList) == 0 {
			this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster))
			return
		}

		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			this.Ui.Error(color.Red("    %+v %s", brokerList, err.Error()))
			return
		}

		topics, err := kfk.Topics()
		swallow(err)
		for _, topic := range topics {
			partions, err := kfk.WritablePartitions(topic)
			swallow(err)
			for _, partitionID := range partions {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)
				host, _, err := net.SplitHostPort(leader.Addr())
				swallow(err)
				if !patternMatched(host, this.hostPattern) {
					continue
				}

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				if err != nil {
					this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err))
					continue
				}
				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				if err != nil {
					this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err))
					continue
				}

				brokerInstances[cluster][host].topicMsgs[topic] += (latestOffset - oldestOffset)
				brokerInstances[cluster][host].addTopicPartition(topic, partitionID)
			}
		}
	})

	hosts := make(map[string]struct{})
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		for host, _ := range brokerInstances[zkcluster.Name()] {
			hosts[host] = struct{}{}
		}
	})
	sortedHosts := make([]string, 0)
	for host, _ := range hosts {
		sortedHosts = append(sortedHosts, host)
	}
	sort.Strings(sortedHosts)

	// sort by host ip
	sortedClusters := make([]string, 0, len(brokerInstances))
	for c, _ := range brokerInstances {
		sortedClusters = append(sortedClusters, c)
	}
	sort.Strings(sortedClusters)

	portN := 0
	hostN := 0
	topicN := 0
	partitionN := 0
	for _, host := range sortedHosts {
		tn := 0
		pn := 0
		mn := int64(0)
		ports := make([]int, 0)
		for _, cluster := range sortedClusters {
			if _, present := brokerInstances[cluster][host]; !present {
				continue
			}

			tn += len(brokerInstances[cluster][host].topicPartitions)
			pn += brokerInstances[cluster][host].leadingPartitions()
			mn += brokerInstances[cluster][host].totalMsgsInStock()
			ports = append(ports, brokerInstances[cluster][host].tcpPorts...)
		}

		portN += len(ports)
		topicN += tn
		partitionN += pn
		hostN += 1

		this.Ui.Output(fmt.Sprintf("  %s leading: %2dT %3dP %15sM ports %2d:%+v",
			color.Green("%15s", host),
			tn,
			pn,
			gofmt.Comma(mn),
			len(ports),
			ports))

		if this.verbose {
			for _, cluster := range sortedClusters {
				if _, present := brokerInstances[cluster][host]; !present {
					continue
				}

				for _, tcpPort := range brokerInstances[cluster][host].tcpPorts {
					this.Ui.Output(fmt.Sprintf("%40d %s", tcpPort,
						gofmt.PrettySince(brokerInstances[cluster][host].uptimes[tcpPort])))
				}
			}

			for _, cluster := range sortedClusters {
				if _, present := brokerInstances[cluster][host]; !present {
					continue
				}

				this.Ui.Output(color.Magenta("%30s", cluster))
				for topic, partitions := range brokerInstances[cluster][host].topicPartitions {
					this.Ui.Output(fmt.Sprintf("%40s: %15sM P%2d %+v",
						topic,
						gofmt.Comma(brokerInstances[cluster][host].topicMsgs[topic]),
						len(partitions), partitions))
				}

			}

		}
	}

	this.Ui.Output(fmt.Sprintf("%17s host:%d, topic:%d, partition:%d, instance:%d",
		"-TOTAL-",
		hostN, topicN, partitionN, portN))
}
Exemple #14
0
func (this *Topics) Run(args []string) (exitCode int) {
	var (
		zone                    string
		cluster                 string
		addTopic                string
		delTopic                string
		killTopic, restoreTopic string
		replicas                int
		partitions              int
		retentionInMinute       int
		resetConf               bool
		debug                   bool
		summaryMode             bool
		configged               bool
	)
	cmdFlags := flag.NewFlagSet("brokers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.topicPattern, "t", "", "")
	cmdFlags.StringVar(&cluster, "c", "", "")
	cmdFlags.BoolVar(&this.verbose, "l", false, "")
	cmdFlags.BoolVar(&this.ipInNumber, "n", false, "")
	cmdFlags.StringVar(&addTopic, "add", "", "")
	cmdFlags.BoolVar(&summaryMode, "sum", false, "")
	cmdFlags.StringVar(&killTopic, "kill", "", "")
	cmdFlags.StringVar(&restoreTopic, "restore", "", "")
	cmdFlags.BoolVar(&this.plainMode, "plain", false, "")
	cmdFlags.StringVar(&delTopic, "del", "", "")
	cmdFlags.IntVar(&partitions, "partitions", 1, "")
	cmdFlags.DurationVar(&this.since, "since", 0, "")
	cmdFlags.BoolVar(&configged, "cf", false, "")
	cmdFlags.BoolVar(&debug, "debug", false, "")
	cmdFlags.BoolVar(&resetConf, "cfreset", false, "")
	cmdFlags.Int64Var(&this.count, "count", 0, "")
	cmdFlags.IntVar(&retentionInMinute, "retention", -1, "")
	cmdFlags.IntVar(&replicas, "replicas", 2, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		on("-add", "-c").
		on("-del", "-c").
		on("-retention", "-c", "-t").
		on("-cfreset", "-c", "-t").
		requireAdminRights("-add", "-del", "-retention").
		invalid(args) {
		return 2
	}

	if this.count > 0 {
		this.verbose = true
	}

	if debug {
		sarama.Logger = log.New(os.Stderr, color.Magenta("[sarama]"), log.LstdFlags)
	}

	if addTopic != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		zkcluster := zkzone.NewCluster(cluster)
		swallow(this.addTopic(zkcluster, addTopic, replicas, partitions))

		return
	} else if delTopic != "" {
		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		zkcluster := zkzone.NewCluster(cluster)
		swallow(this.delTopic(zkcluster, delTopic))

		return
	}

	ensureZoneValid(zone)

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	if retentionInMinute > 0 {
		zkcluster := zkzone.NewCluster(cluster)
		this.configTopic(zkcluster, this.topicPattern, retentionInMinute)
		return
	}

	if resetConf {
		zkcluster := zkzone.NewCluster(cluster)
		this.resetTopicConfig(zkcluster, this.topicPattern)
		configged = true // after reset, display most recent znode info
	}

	if configged {
		// output header
		this.Ui.Output(fmt.Sprintf("%25s %-40s %15s", "cluster", "topic", "mtime"))
		this.Ui.Output(fmt.Sprintf("%25s %40s %15s",
			strings.Repeat("-", 25), strings.Repeat("-", 40), strings.Repeat("-", 15)))

		displayTopicConfigs := func(zkcluster *zk.ZkCluster) {
			// sort by topic name
			configs := zkcluster.ConfiggedTopics()
			sortedTopics := make([]string, 0, len(configs))
			for topic, _ := range configs {
				sortedTopics = append(sortedTopics, topic)
			}
			sort.Strings(sortedTopics)

			for _, topic := range sortedTopics {
				if !patternMatched(topic, this.topicPattern) {
					continue
				}

				configInfo := configs[topic]
				this.Ui.Output(fmt.Sprintf("%25s %40s %15s %s",
					zkcluster.Name(),
					topic,
					gofmt.PrettySince(configInfo.Mtime),
					configInfo.Config))
			}

		}

		if cluster == "" {
			zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				displayTopicConfigs(zkcluster)
			})
		} else {
			zkcluster := zkzone.NewCluster(cluster)
			displayTopicConfigs(zkcluster)
		}

		return
	}

	if summaryMode {
		this.printSummary(zkzone, cluster)
		return
	}

	if !this.verbose {
		// output header
		this.Ui.Output(fmt.Sprintf("%30s %-50s", "cluster", "topic"))
		this.Ui.Output(fmt.Sprintf("%30s %50s",
			strings.Repeat("-", 30), strings.Repeat("-", 50)))
	}

	if cluster != "" {
		zkcluster := zkzone.NewCluster(cluster)
		this.displayTopicsOfCluster(zkcluster)

		this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Topics-", this.topicN))
		this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Partitions-", this.partitionN))
		if this.verbose {
			if this.plainMode {
				fmt.Printf("%25s %s\n", "-FLAT Messages-", gofmt.Comma(this.totalMsgs))
				fmt.Printf("%25s %s\n", "-CUM Messages-", gofmt.Comma(this.totalOffsets))
			} else {
				this.Ui.Output(fmt.Sprintf("%25s %s", "-FLAT Messages-", gofmt.Comma(this.totalMsgs)))
				this.Ui.Output(fmt.Sprintf("%25s %s", "-CUM Messages-", gofmt.Comma(this.totalOffsets)))
			}

		}
		return
	}

	// all clusters
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		this.displayTopicsOfCluster(zkcluster)
	})
	this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Topics-", this.topicN))
	this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Partitions-", this.partitionN))
	if this.verbose {
		if this.plainMode {
			fmt.Printf("%25s %s\n", "-FLAT Messages-", gofmt.Comma(this.totalMsgs))
			fmt.Printf("%25s %s\n", "-CUM Messages-", gofmt.Comma(this.totalOffsets))
		} else {
			this.Ui.Output(fmt.Sprintf("%25s %s", "-FLAT Messages-", gofmt.Comma(this.totalMsgs)))
			this.Ui.Output(fmt.Sprintf("%25s %s", "-CUM Messages-", gofmt.Comma(this.totalOffsets)))
		}

	}

	return
}
Exemple #15
0
func (this *Topics) displayTopicsOfCluster(zkcluster *zk.ZkCluster) {
	echoBuffer := func(lines []string) {
		for _, l := range lines {
			this.Ui.Output(l)
		}
	}

	linesInTopicMode := make([]string, 0)
	if this.verbose {
		linesInTopicMode = this.echoOrBuffer(zkcluster.Name(), linesInTopicMode)
	}

	// get all alive brokers within this cluster
	brokers := zkcluster.Brokers()
	if len(brokers) == 0 {
		linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s", " ",
			color.Red("%s empty brokers", zkcluster.Name())), linesInTopicMode)
		echoBuffer(linesInTopicMode)
		return
	}

	if this.verbose {
		sortedBrokerIds := make([]string, 0, len(brokers))
		for brokerId, _ := range brokers {
			sortedBrokerIds = append(sortedBrokerIds, brokerId)
		}
		sort.Strings(sortedBrokerIds)
		for _, brokerId := range sortedBrokerIds {
			if this.ipInNumber {
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ",
					color.Green(brokerId), brokers[brokerId]), linesInTopicMode)
			} else {
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ",
					color.Green(brokerId), brokers[brokerId].NamedString()), linesInTopicMode)
			}

		}
	}

	kfk, err := sarama.NewClient(zkcluster.BrokerList(), saramaConfig())
	if err != nil {
		if this.verbose {
			linesInTopicMode = this.echoOrBuffer(color.Yellow("%5s%+v %s", " ",
				zkcluster.BrokerList(), err.Error()), linesInTopicMode)
		}

		return
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		if this.topicPattern == "" && this.verbose {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%5s%s", " ",
				color.Magenta("no topics")), linesInTopicMode)
			echoBuffer(linesInTopicMode)
		}

		return
	}

	sortedTopics := make([]string, 0, len(topics))
	for _, t := range topics {
		sortedTopics = append(sortedTopics, t)
	}
	sort.Strings(sortedTopics)

	topicsCtime := zkcluster.TopicsCtime()
	hasTopicMatched := false
	for _, topic := range sortedTopics {
		if !patternMatched(topic, this.topicPattern) {
			continue
		}

		if this.since > 0 && time.Since(topicsCtime[topic]) > this.since {
			continue
		}

		this.topicN++

		hasTopicMatched = true
		if this.verbose {
			linesInTopicMode = this.echoOrBuffer(strings.Repeat(" ", 4)+color.Cyan(topic), linesInTopicMode)
		}

		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		swallow(err)
		partions, err := kfk.Partitions(topic)
		swallow(err)
		if len(alivePartitions) != len(partions) {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %s P: %s/%+v",
				zkcluster.Name(), color.Cyan("%-50s", topic), color.Red("partial dead"), color.Green("%+v", alivePartitions), partions), linesInTopicMode)
		}

		replicas, err := kfk.Replicas(topic, partions[0])
		if err != nil {
			this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partions[0], err))
		}

		this.partitionN += len(partions)
		if !this.verbose {
			linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %3dP %dR %s",
				zkcluster.Name(),
				color.Cyan("%-50s", topic),
				len(partions), len(replicas),
				gofmt.PrettySince(topicsCtime[topic])), linesInTopicMode)
			continue
		}

		for _, partitionID := range alivePartitions {
			leader, err := kfk.Leader(topic, partitionID)
			swallow(err)

			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partitionID, err))
			}

			isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID)
			isrMtimeSince := gofmt.PrettySince(isrMtime)
			if time.Since(isrMtime).Hours() < 24 {
				// ever out of sync last 24h
				isrMtimeSince = color.Magenta(isrMtimeSince)
			}

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			latestOffset, err := kfk.GetOffset(topic, partitionID,
				sarama.OffsetNewest)
			swallow(err)

			oldestOffset, err := kfk.GetOffset(topic, partitionID,
				sarama.OffsetOldest)
			swallow(err)

			if this.count > 0 && (latestOffset-oldestOffset) < this.count {
				continue
			}

			this.totalMsgs += latestOffset - oldestOffset
			this.totalOffsets += latestOffset
			if !underReplicated {
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%+v Offset:%16s - %-16s Num:%-15s %s-%s",
					partitionID,
					color.Green("%d", leader.ID()), replicas, isr,
					gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset),
					gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode)
			} else {
				// use red for alert
				linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%s Offset:%16s - %-16s Num:%-15s %s-%s",
					partitionID,
					color.Green("%d", leader.ID()), replicas, color.Red("%+v", isr),
					gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset),
					gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode)
			}
		}
	}

	if this.topicPattern != "" {
		if hasTopicMatched {
			echoBuffer(linesInTopicMode)
		}

	} else {
		echoBuffer(linesInTopicMode)
	}
}
Exemple #16
0
func (this *Consumers) printConsumersByGroupTable(zkzone *zk.ZkZone, clusterPattern string) {
	lines := make([]string, 0)
	header := "Zone|Cluster|M|Host|ConsumerGroup|Topic/Partition|Offset|Uptime"
	lines = append(lines, header)

	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		groupTopicsMap := make(map[string]map[string]struct{}) // group:sub topics

		if !patternMatched(zkcluster.Name(), clusterPattern) {
			return
		}

		consumerGroups := zkcluster.ConsumerGroups()
		sortedGroups := make([]string, 0, len(consumerGroups))
		for group, _ := range consumerGroups {
			if !patternMatched(group, this.groupPattern) {
				continue
			}

			sortedGroups = append(sortedGroups, group)
		}

		sort.Strings(sortedGroups)
		for _, group := range sortedGroups {
			consumers := consumerGroups[group]
			if this.onlineOnly && len(consumers) == 0 {
				continue
			}

			if len(consumers) > 0 {
				// sort by host
				sortedIds := make([]string, 0)
				consumersMap := make(map[string]*zk.ConsumerZnode)
				for _, c := range consumers {
					sortedIds = append(sortedIds, c.Id)
					consumersMap[c.Id] = c
				}
				sort.Strings(sortedIds)

				for _, consumerId := range sortedIds {
					c := consumersMap[consumerId]
					for topic, _ := range c.Subscription {
						if !patternMatched(topic, this.topicPattern) {
							continue
						}

						if len(groupTopicsMap[group]) == 0 {
							groupTopicsMap[group] = make(map[string]struct{}, 5)
						}
						groupTopicsMap[group][topic] = struct{}{}

						ownerByPartition := zkcluster.OwnersOfGroupByTopic(group, topic)

						partitionsWithOffset := make(map[string]struct{})
						for _, offset := range this.displayGroupOffsets(zkcluster, group, topic, false) {

							onlineSymbol := "◉"
							isOwner := false
							if ownerByPartition[offset.partitionId] == consumerId {
								onlineSymbol += "*" // owned by this consumer
								isOwner = true
							}
							if this.ownerOnly && !isOwner {
								continue
							}

							partitionsWithOffset[offset.partitionId] = struct{}{}

							lines = append(lines,
								fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s|%s",
									zkzone.Name(), zkcluster.Name(),
									onlineSymbol,
									c.Host(),
									group+"@"+c.Id[len(c.Id)-12:],
									fmt.Sprintf("%s/%s", offset.topic, offset.partitionId),
									offset.offset,
									gofmt.PrettySince(c.Uptime())))
						}

						for partitionId, _ := range ownerByPartition {
							if _, present := partitionsWithOffset[partitionId]; !present {
								// this consumer is owner online, but has no offset
								onlineSymbol := "◉"
								isOwner := false
								if ownerByPartition[partitionId] == consumerId {
									onlineSymbol += "*"
									isOwner = true
								}
								if this.ownerOnly && !isOwner {
									continue
								}

								lines = append(lines,
									fmt.Sprintf("%s|%s|%s|%s|%s|%s|?|%s",
										zkzone.Name(), zkcluster.Name(),
										onlineSymbol,
										c.Host(),
										group+"@"+c.Id[len(c.Id)-12:],
										fmt.Sprintf("%s/%s", topic, partitionId),
										gofmt.PrettySince(c.Uptime())))
							}
						}
					}

				}
			} else {
				// offline
				for _, offset := range this.displayGroupOffsets(zkcluster, group, "", false) {
					if !patternMatched(offset.topic, this.topicPattern) {
						continue
					}

					lines = append(lines,
						fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s|%s",
							zkzone.Name(), zkcluster.Name(),
							"◎",
							" ",
							group, fmt.Sprintf("%s/%s", offset.topic, offset.partitionId),
							offset.offset, " "))
				}
			}
		}

		for group, topics := range groupTopicsMap {
			if len(topics) > 1 {
				// the same consumer group is consuming more than 1 topics
				topicsLabel := make([]string, 0, len(topics))
				for t, _ := range topics {
					topicsLabel = append(topicsLabel, t)
				}
				this.Ui.Warn(fmt.Sprintf("%35s consuming: %+v", group, topicsLabel))
			}
		}
	})

	if !this.warnOnly {
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

}