func (this *Pps) showPps(nic string) { tx := fmt.Sprintf("/sys/class/net/%s/statistics/tx_packets", nic) rx := fmt.Sprintf("/sys/class/net/%s/statistics/rx_packets", nic) var lastTx, lastRx int64 s := int64(this.interval.Seconds()) for { brx, err := ioutil.ReadFile(rx) swallow(err) btx, err := ioutil.ReadFile(tx) swallow(err) rxN, err := strconv.ParseInt(strings.TrimSpace(string(brx)), 10, 64) swallow(err) txN, err := strconv.ParseInt(strings.TrimSpace(string(btx)), 10, 64) swallow(err) if lastRx != 0 && lastTx != 0 { rxPps := (rxN - lastRx) / s txPps := (txN - lastTx) / s sumPps := rxPps + txPps this.Ui.Output(fmt.Sprintf("%10s rx:%-8s tx:%-8s sum:%-8s", nic, gofmt.Comma(rxPps), gofmt.Comma(txPps), gofmt.Comma(sumPps))) } lastRx = rxN lastTx = txN time.Sleep(this.interval) } }
func (this *Clusters) printSummary(zkzone *zk.ZkZone, clusterPattern string, port string) { lines := []string{"Zone|Cluster|Brokers|Topics|Partitions|FlatMsg|Cum"} type summary struct { zone, cluster string brokers, topics, partitions int flat, cum int64 } summaries := make([]summary, 0, 10) zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { if !patternMatched(zkcluster.Name(), clusterPattern) { return } brokers, topics, partitions, flat, cum := this.clusterSummary(zkcluster) summaries = append(summaries, summary{zkzone.Name(), zkcluster.Name(), brokers, topics, partitions, flat, cum}) }) sortutil.DescByField(summaries, "cum") var totalFlat, totalCum int64 for _, s := range summaries { lines = append(lines, fmt.Sprintf("%s|%s|%d|%d|%d|%s|%s", s.zone, s.cluster, s.brokers, s.topics, s.partitions, gofmt.Comma(s.flat), gofmt.Comma(s.cum))) totalCum += s.cum totalFlat += s.flat } this.Ui.Output(columnize.SimpleFormat(lines)) this.Ui.Output(fmt.Sprintf("Flat:%s Cum:%s", gofmt.Comma(totalFlat), gofmt.Comma(totalCum))) }
func (this *FunServantImpl) showStats() { ticker := time.NewTicker(config.Engine.Servants.StatsOutputInterval) defer ticker.Stop() for _ = range ticker.C { log.Info("svt: {slow:%s peer.from:%s, peer.to:%s}", gofmt.Comma(svtStats.callsSlow), gofmt.Comma(svtStats.callsFromPeer), gofmt.Comma(svtStats.callsToPeer)) } }
func (this *routerStats) render(logger *log.Logger, elapsed int) { logger.Printf("Total:%10s %10s speed:%6s/s %10s/s max: %s/%s", gofmt.Comma(this.TotalProcessedMsgN), gofmt.ByteSize(this.TotalProcessedBytes), gofmt.Comma(int64(this.PeriodProcessedMsgN/int32(elapsed))), gofmt.ByteSize(this.PeriodProcessedBytes/int64(elapsed)), gofmt.ByteSize(this.PeriodMaxMsgBytes), gofmt.ByteSize(this.TotalMaxMsgBytes)) logger.Printf("Input:%10s %10s speed:%6s/s %10s/s", gofmt.Comma(int64(this.PeriodInputMsgN)), gofmt.ByteSize(this.PeriodInputBytes), gofmt.Comma(int64(this.PeriodInputMsgN/int32(elapsed))), gofmt.ByteSize(this.PeriodInputBytes/int64(elapsed))) }
func (this *Histogram) showOffsetGrowth() ([]time.Time, []int64) { f, err := os.OpenFile(this.offsetFile, os.O_RDONLY, 0660) swallow(err) defer f.Close() ts := make([]time.Time, 0) vs := make([]int64, 0) r := bufio.NewReader(f) var ( lastN = int64(0) tm string ) for { line, err := r.ReadString('\n') if err == io.EOF { break } line = strings.TrimSpace(line) if !strings.Contains(line, "CUM Messages") { // time info: Thu Jun 16 22:45:01 CST 2016 tm = line } else { // offset: -CUM Messages- 255,705,684,384 n := strings.Split(line, "-CUM Messages-")[1] n = strings.Replace(n, ",", "", -1) n = strings.TrimSpace(n) offset, err := strconv.ParseInt(n, 10, 64) swallow(err) if lastN > 0 { t, e := time.Parse("Mon Jan 2 15:04:05 MST 2006", tm) swallow(e) ts = append(ts, t) vs = append(vs, offset-lastN) this.Ui.Output(fmt.Sprintf("%55s Message+ %15s/%s", tm, gofmt.Comma(offset-lastN), gofmt.Comma(lastN))) } lastN = offset } } return ts, vs }
func (this *TFunServer) showStats(interval time.Duration) { ticker := time.NewTicker(interval) defer ticker.Stop() for _ = range ticker.C { log.Info("rpc: {session.on:%d/%s, call.err:%s/%s, qps:{1m:%.0f, 5m:%.0f 15m:%.0f avg:%.0f}}", atomic.LoadInt64(&this.activeSessionN), gofmt.Comma(atomic.LoadInt64(&this.cumSessions)), gofmt.Comma(atomic.LoadInt64(&this.cumCallErrs)), gofmt.Comma(atomic.LoadInt64(&this.cumCalls)), this.stats.CallPerSecond.Rate1(), this.stats.CallPerSecond.Rate5(), this.stats.CallPerSecond.Rate15(), this.stats.CallPerSecond.RateMean()) } }
// @rest GET /v1/status func (this *manServer) statusHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) { log.Info("status %s(%s)", r.RemoteAddr, getHttpRemoteIp(r)) output := make(map[string]interface{}) output["options"] = Options output["loglevel"] = logLevel.String() output["manager"] = manager.Default.Dump() pubConns := int(atomic.LoadInt32(&this.gw.pubServer.activeConnN)) subConns := int(atomic.LoadInt32(&this.gw.subServer.activeConnN)) output["pubconn"] = strconv.Itoa(pubConns) output["subconn"] = strconv.Itoa(subConns) output["hh_appends"] = strconv.FormatInt(hh.Default.AppendN(), 10) output["hh_delivers"] = strconv.FormatInt(hh.Default.DeliverN(), 10) output["goroutines"] = strconv.Itoa(runtime.NumGoroutine()) var mem runtime.MemStats runtime.ReadMemStats(&mem) output["heap"] = gofmt.ByteSize(mem.HeapSys).String() output["objects"] = gofmt.Comma(int64(mem.HeapObjects)) b, err := json.MarshalIndent(output, "", " ") if err != nil { log.Error("%s(%s) %v", r.RemoteAddr, getHttpRemoteIp(r), err) } w.Write(b) }
func main() { cf := config.NewDefaultProxy() cf.IoTimeout = time.Hour cf.TcpNoDelay = tcpNoDelay prx := proxy.New(cf) etclib.Dial([]string{zk}) go prx.StartMonitorCluster() prx.AwaitClusterTopologyReady() // test pool if testPool { testServantPool(prx) pause("pool tested") } go report.run() wg := new(sync.WaitGroup) t1 := time.Now() for k := c1; k <= c2; k += 10 { Concurrency = k cf.PoolCapacity = Concurrency prx = proxy.New(cf) for i := 0; i < Rounds; i++ { for j := 0; j < k; j++ { wg.Add(1) go runSession(prx, wg, i+1, j) } wg.Wait() } } elapsed := time.Since(t1) log.Printf("Elapsed: %s, calls: {%s, %.1f/s}, sessions: {%s, %.1f/s}, errors: {conn:%d, io:%d call:%d}", elapsed, gofmt.Comma(report.callOk), float64(report.callOk)/elapsed.Seconds(), gofmt.Comma(int64(report.sessionN)), float64(report.sessionN)/elapsed.Seconds(), report.connErrs, report.ioErrs, report.callErrs) }
func (this *Haproxy) getStats(statsUri string) (header string, rows []string) { client := http.Client{Timeout: time.Second * 30} resp, err := client.Get(statsUri) swallow(err) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { swallow(fmt.Errorf("fetch[%s] stats got status: %d", resp.StatusCode)) } var records map[string]map[string]int64 reader := json.NewDecoder(resp.Body) err = reader.Decode(&records) swallow(err) u, err := url.Parse(statsUri) swallow(err) var shortHostname string if strings.Contains(u.Host, ":") { u.Host = u.Host[:strings.Index(u.Host, ":")] } tuples := strings.SplitN(u.Host, ".", 4) if len(tuples) < 4 { shortHostname = u.Host } else { shortHostname = tuples[3] } if len(shortHostname) > 8 { shortHostname = shortHostname[:8] } sortedSvcs := make([]string, 0) for svc, _ := range records { sortedSvcs = append(sortedSvcs, svc) } sort.Strings(sortedSvcs) sortedCols := make([]string, 0) for k, _ := range records["pub"] { sortedCols = append(sortedCols, k) } sort.Strings(sortedCols) header = strings.Join(append([]string{"host", "svc"}, sortedCols...), "|") for _, svc := range sortedSvcs { stats := records[svc] var vals = []string{shortHostname, svc} for _, k := range sortedCols { v := stats[k] vals = append(vals, gofmt.Comma(v)) } rows = append(rows, strings.Join(vals, "|")) } return }
func (this *Mirror) pump(sub *consumergroup.ConsumerGroup, pub sarama.AsyncProducer, stop chan struct{}) { defer func() { log.Println("pump cleanup...") sub.Close() log.Println("pump cleanup ok") stop <- struct{}{} // notify others I'm done }() log.Printf("start pumping") active := false for { select { case <-this.quit: return case <-stop: // yes sir! return case <-time.After(time.Second * 10): active = false log.Println("idle 10s waiting for new msg") case msg := <-sub.Messages(): if !active || this.debug { log.Printf("<-[%d] T:%s M:%s", this.transferN, msg.Topic, string(msg.Value)) } active = true pub.Input() <- &sarama.ProducerMessage{ Topic: msg.Topic, Key: sarama.ByteEncoder(msg.Key), Value: sarama.ByteEncoder(msg.Value), } if this.autoCommit { sub.CommitUpto(msg) } // rate limit, never overflood the limited bandwidth between IDCs // FIXME when compressed, the bandwidth calculation is wrong bytesN := len(msg.Topic) + len(msg.Key) + len(msg.Value) + 20 // payload overhead if !this.bandwidthRateLimiter.Pour(bytesN) { time.Sleep(time.Second) this.Ui.Warn(fmt.Sprintf("%d -> bandwidth reached, backoff 1s", bytesN)) } this.transferBytes += int64(bytesN) this.transferN++ if this.transferN%this.progressStep == 0 { log.Println(gofmt.Comma(this.transferN)) } case err := <-sub.Errors(): this.Ui.Error(err.Error()) // TODO } } }
func (this *Mirror) makeMirror(c1, c2 *zk.ZkCluster) { pub, err := this.makePub(c2) swallow(err) topics, topicsChanges, err := c1.WatchTopics() swallow(err) log.Printf("topics: %+v", topics) if len(topics) == 0 { log.Println("empty topics") return } group := fmt.Sprintf("%s.%s._mirror_", c1.Name(), c2.Name()) sub, err := this.makeSub(c1, group, topics) swallow(err) pumpStopper := make(chan struct{}) go this.pump(sub, pub, pumpStopper) LOOP: for { select { case <-topicsChanges: log.Println("topics changed, stopping pump...") pumpStopper <- struct{}{} // stop pump <-pumpStopper // await pump cleanup // refresh c1 topics topics, err = c1.Topics() if err != nil { // TODO how to handle this err? log.Println(err) } log.Printf("topics: %+v", topics) sub, err = this.makeSub(c1, group, topics) if err != nil { // TODO how to handle this err? log.Println(err) } go this.pump(sub, pub, pumpStopper) case <-this.quit: log.Println("awaiting pump cleanup...") <-pumpStopper log.Printf("total transferred: %s %smsgs", gofmt.ByteSize(this.transferBytes), gofmt.Comma(this.transferN)) break LOOP } } pub.Close() }
func (this *Mirror) Run(args []string) (exitCode int) { cmdFlags := flag.NewFlagSet("mirror", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&this.zone1, "z1", "", "") cmdFlags.StringVar(&this.zone2, "z2", "", "") cmdFlags.StringVar(&this.cluster1, "c1", "", "") cmdFlags.StringVar(&this.cluster2, "c2", "", "") cmdFlags.StringVar(&this.excludes, "excluded", "", "") cmdFlags.BoolVar(&this.debug, "debug", false, "") cmdFlags.StringVar(&this.compress, "compress", "", "") cmdFlags.Int64Var(&this.bandwidthLimit, "net", 100, "") cmdFlags.BoolVar(&this.autoCommit, "commit", true, "") cmdFlags.Int64Var(&this.progressStep, "step", 5000, "") if err := cmdFlags.Parse(args); err != nil { return 1 } if validateArgs(this, this.Ui). require("-z1", "-z2", "-c1", "-c2"). invalid(args) { return 2 } this.topicsExcluded = make(map[string]struct{}) for _, e := range strings.Split(this.excludes, ",") { this.topicsExcluded[e] = struct{}{} } log.SetOutput(os.Stdout) this.quit = make(chan struct{}) limit := (1 << 20) * this.bandwidthLimit / 8 this.bandwidthRateLimiter = ratelimiter.NewLeakyBucket(limit, time.Second) log.Printf("[%s]%s -> [%s]%s with bandwidth %sbps", this.zone1, this.cluster1, this.zone2, this.cluster2, gofmt.Comma(int64(limit*8))) signal.RegisterSignalsHandler(func(sig os.Signal) { log.Printf("received signal: %s", strings.ToUpper(sig.String())) log.Println("quiting...") this.once.Do(func() { close(this.quit) }) }, syscall.SIGINT, syscall.SIGTERM) z1 := zk.NewZkZone(zk.DefaultConfig(this.zone1, ctx.ZoneZkAddrs(this.zone1))) z2 := zk.NewZkZone(zk.DefaultConfig(this.zone2, ctx.ZoneZkAddrs(this.zone2))) c1 := z1.NewCluster(this.cluster1) c2 := z2.NewCluster(this.cluster2) this.makeMirror(c1, c2) return }
func (this *stats) run() { ticker := time.NewTicker(time.Second) defer ticker.Stop() t1 := time.Now() var lastCalls int64 for _ = range ticker.C { if neatStat { log.Printf("c:%6d qps:%20s errs:%10s", Concurrency, gofmt.Comma(this.callOk-lastCalls), gofmt.Comma(this.callErrs)) } else { log.Printf("%s c:%d sessions:%s calls:%s qps:%s errs:%s conns:%d go:%d", time.Since(t1), Concurrency, gofmt.Comma(int64(atomic.LoadInt32(&this.sessionN))), gofmt.Comma(atomic.LoadInt64(&this.callOk)), gofmt.Comma(this.callOk-lastCalls), gofmt.Comma(this.callErrs), atomic.LoadInt32(&this.concurrentN), runtime.NumGoroutine()) } lastCalls = this.callOk } }
func (this *Consumers) displayGroupOffsets(zkcluster *zk.ZkCluster, group, topic string, echo bool) []consumerGroupOffset { offsetMap := zkcluster.ConsumerOffsetsOfGroup(group) sortedTopics := make([]string, 0, len(offsetMap)) for t, _ := range offsetMap { sortedTopics = append(sortedTopics, t) } sort.Strings(sortedTopics) r := make([]consumerGroupOffset, 0) for _, t := range sortedTopics { if !patternMatched(t, this.topicPattern) || (topic != "" && t != topic) { continue } sortedPartitionIds := make([]string, 0, len(offsetMap[t])) for partitionId, _ := range offsetMap[t] { sortedPartitionIds = append(sortedPartitionIds, partitionId) } sort.Strings(sortedPartitionIds) for _, partitionId := range sortedPartitionIds { r = append(r, consumerGroupOffset{ topic: t, partitionId: partitionId, offset: gofmt.Comma(offsetMap[t][partitionId]), }) if echo { this.Ui.Output(fmt.Sprintf("\t\t%s/%s Offset:%s", t, partitionId, gofmt.Comma(offsetMap[t][partitionId]))) } } } return r }
func (this *EsOutput) showPeriodicalStats() { if !this.showProgress { return } var ( globals = engine.Globals() total = 0 ) globals.Printf("ES types: %d, within %s", this.counters.Len(), this.reportInterval) for _, key := range this.counters.SortedKeys() { val := this.counters.Get(key) if val > 0 { total += val globals.Printf("%-50s %12s", key, gofmt.Comma(int64(val))) this.counters.Set(key, 0) } } globals.Printf("%50s %12s", "Sum", gofmt.Comma(int64(total))) }
func RunSysStats(startedAt time.Time, interval time.Duration) { const nsInMs uint64 = 1000 * 1000 ticker := time.NewTicker(interval) defer func() { ticker.Stop() }() var ( ms = new(runtime.MemStats) rusage = &syscall.Rusage{} lastUserTime int64 lastSysTime int64 userTime int64 sysTime int64 userCpuUtil float64 sysCpuUtil float64 ) for _ = range ticker.C { runtime.ReadMemStats(ms) syscall.Getrusage(syscall.RUSAGE_SELF, rusage) syscall.Getrusage(syscall.RUSAGE_SELF, rusage) userTime = rusage.Utime.Sec*1000000000 + int64(rusage.Utime.Usec) sysTime = rusage.Stime.Sec*1000000000 + int64(rusage.Stime.Usec) userCpuUtil = float64(userTime-lastUserTime) * 100 / float64(interval) sysCpuUtil = float64(sysTime-lastSysTime) * 100 / float64(interval) lastUserTime = userTime lastSysTime = sysTime log.Info("ver:%s, since:%s, go:%d, gc:%dms/%d=%d, heap:{%s, %s, %s, %s %s} cpu:{%3.2f%%us, %3.2f%%sy}", BuildId, time.Since(startedAt), runtime.NumGoroutine(), ms.PauseTotalNs/nsInMs, ms.NumGC, ms.PauseTotalNs/(nsInMs*uint64(ms.NumGC))+1, gofmt.ByteSize(ms.HeapSys), // bytes it has asked the operating system for gofmt.ByteSize(ms.HeapAlloc), // bytes currently allocated in the heap gofmt.ByteSize(ms.HeapIdle), // bytes in the heap that are unused gofmt.ByteSize(ms.HeapReleased), // bytes returned to the operating system, 5m for scavenger gofmt.Comma(int64(ms.HeapObjects)), userCpuUtil, sysCpuUtil) } }
func (this *Topics) printSummary(zkzone *zk.ZkZone, clusterPattern string) { lines := []string{"Zone|Cluster|Topic|Partitions|FlatMsg|Cum"} var totalFlat, totalCum int64 zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { if !patternMatched(zkcluster.Name(), clusterPattern) { return } summaries := this.clusterSummary(zkcluster) sortutil.DescByField(summaries, "cum") for _, s := range summaries { lines = append(lines, fmt.Sprintf("%s|%s|%s|%d|%s|%s", s.zone, s.cluster, s.topic, s.partitions, gofmt.Comma(s.flat), gofmt.Comma(s.cum))) totalCum += s.cum totalFlat += s.flat } }) this.Ui.Output(columnize.SimpleFormat(lines)) this.Ui.Output(fmt.Sprintf("Flat:%s Cum:%s", gofmt.Comma(totalFlat), gofmt.Comma(totalCum))) }
func (this *Haproxy) fetchStats(statsUri string) { client := http.Client{Timeout: time.Second * 30} resp, err := client.Get(statsUri) swallow(err) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { swallow(fmt.Errorf("fetch[%s] stats got status: %d", resp.StatusCode)) } var records map[string]map[string]int64 reader := json.NewDecoder(resp.Body) err = reader.Decode(&records) swallow(err) u, err := url.Parse(statsUri) swallow(err) this.Ui.Info(u.Host) sortedSvcs := make([]string, 0) for svc, _ := range records { sortedSvcs = append(sortedSvcs, svc) } sort.Strings(sortedSvcs) sortedCols := make([]string, 0) for k, _ := range records["pub"] { sortedCols = append(sortedCols, k) } sort.Strings(sortedCols) lines := []string{strings.Join(append([]string{"svc"}, sortedCols...), "|")} for _, svc := range sortedSvcs { stats := records[svc] var vals = []string{svc} for _, k := range sortedCols { v := stats[k] vals = append(vals, gofmt.Comma(v)) } lines = append(lines, strings.Join(vals, "|")) } this.Ui.Output(columnize.SimpleFormat(lines)) }
func (this *engineStats) Runtime() map[string]interface{} { runtime.ReadMemStats(this.memStats) s := make(map[string]interface{}) s["goroutines"] = runtime.NumGoroutine() s["memory.allocated"] = gofmt.ByteSize(this.memStats.Alloc).String() s["memory.mallocs"] = gofmt.ByteSize(this.memStats.Mallocs).String() s["memory.frees"] = gofmt.ByteSize(this.memStats.Frees).String() s["memory.last_gc"] = this.memStats.LastGC s["memory.gc.num"] = this.memStats.NumGC s["memory.gc.num_per_second"] = float64(this.memStats.NumGC) / time. Since(this.startedAt).Seconds() s["memory.gc.num_freq"] = fmt.Sprintf("%.1fsec/gc", time. Since(this.startedAt).Seconds()/float64(this.memStats.NumGC)) s["memory.gc.total_pause"] = fmt.Sprintf("%dms", this.memStats.PauseTotalNs/uint64(time.Millisecond)) s["memory.heap.alloc"] = gofmt.ByteSize(this.memStats.HeapAlloc).String() s["memory.heap.sys"] = gofmt.ByteSize(this.memStats.HeapSys).String() s["memory.heap.idle"] = gofmt.ByteSize(this.memStats.HeapIdle).String() s["memory.heap.released"] = gofmt.ByteSize(this.memStats.HeapReleased).String() s["memory.heap.objects"] = gofmt.Comma(int64(this.memStats.HeapObjects)) s["memory.stack"] = gofmt.ByteSize(this.memStats.StackInuse).String() gcPausesMs := make([]string, 0, 20) for _, pauseNs := range this.memStats.PauseNs { if pauseNs == 0 { continue } pauseStr := fmt.Sprintf("%dms", pauseNs/uint64(time.Millisecond)) if pauseStr == "0ms" { continue } gcPausesMs = append(gcPausesMs, pauseStr) } s["memory.gc.pauses"] = gcPausesMs s["mem"] = *this.memStats return s }
func (this *Topology) displayZoneTopology(zkzone *zk.ZkZone) { this.Ui.Output(zkzone.Name()) // {cluster: {topic: brokerHostInfo}} brokerInstances := make(map[string]map[string]*brokerHostInfo) zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) { if len(liveBrokers) == 0 { this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster)) return } if this.cluster != "" && this.cluster != cluster { return } brokerInstances[cluster] = make(map[string]*brokerHostInfo) for _, broker := range liveBrokers { if !patternMatched(broker.Host, this.hostPattern) { continue } if _, present := brokerInstances[cluster][broker.Host]; !present { brokerInstances[cluster][broker.Host] = newBrokerHostInfo() } brokerInstances[cluster][broker.Host].addPort(broker.Port, broker.Uptime()) } // find how many partitions a broker is leading zkcluster := zkzone.NewCluster(cluster) brokerList := zkcluster.BrokerList() if len(brokerList) == 0 { this.Ui.Warn(fmt.Sprintf("empty brokers in cluster[%s]", cluster)) return } kfk, err := sarama.NewClient(brokerList, sarama.NewConfig()) if err != nil { this.Ui.Error(color.Red(" %+v %s", brokerList, err.Error())) return } topics, err := kfk.Topics() swallow(err) for _, topic := range topics { partions, err := kfk.WritablePartitions(topic) swallow(err) for _, partitionID := range partions { leader, err := kfk.Leader(topic, partitionID) swallow(err) host, _, err := net.SplitHostPort(leader.Addr()) swallow(err) if !patternMatched(host, this.hostPattern) { continue } latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) if err != nil { this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err)) continue } oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest) if err != nil { this.Ui.Error(fmt.Sprintf("%s %s %v", cluster, topic, err)) continue } brokerInstances[cluster][host].topicMsgs[topic] += (latestOffset - oldestOffset) brokerInstances[cluster][host].addTopicPartition(topic, partitionID) } } }) hosts := make(map[string]struct{}) zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { for host, _ := range brokerInstances[zkcluster.Name()] { hosts[host] = struct{}{} } }) sortedHosts := make([]string, 0) for host, _ := range hosts { sortedHosts = append(sortedHosts, host) } sort.Strings(sortedHosts) // sort by host ip sortedClusters := make([]string, 0, len(brokerInstances)) for c, _ := range brokerInstances { sortedClusters = append(sortedClusters, c) } sort.Strings(sortedClusters) portN := 0 hostN := 0 topicN := 0 partitionN := 0 for _, host := range sortedHosts { tn := 0 pn := 0 mn := int64(0) ports := make([]int, 0) for _, cluster := range sortedClusters { if _, present := brokerInstances[cluster][host]; !present { continue } tn += len(brokerInstances[cluster][host].topicPartitions) pn += brokerInstances[cluster][host].leadingPartitions() mn += brokerInstances[cluster][host].totalMsgsInStock() ports = append(ports, brokerInstances[cluster][host].tcpPorts...) } portN += len(ports) topicN += tn partitionN += pn hostN += 1 this.Ui.Output(fmt.Sprintf(" %s leading: %2dT %3dP %15sM ports %2d:%+v", color.Green("%15s", host), tn, pn, gofmt.Comma(mn), len(ports), ports)) if this.verbose { for _, cluster := range sortedClusters { if _, present := brokerInstances[cluster][host]; !present { continue } for _, tcpPort := range brokerInstances[cluster][host].tcpPorts { this.Ui.Output(fmt.Sprintf("%40d %s", tcpPort, gofmt.PrettySince(brokerInstances[cluster][host].uptimes[tcpPort]))) } } for _, cluster := range sortedClusters { if _, present := brokerInstances[cluster][host]; !present { continue } this.Ui.Output(color.Magenta("%30s", cluster)) for topic, partitions := range brokerInstances[cluster][host].topicPartitions { this.Ui.Output(fmt.Sprintf("%40s: %15sM P%2d %+v", topic, gofmt.Comma(brokerInstances[cluster][host].topicMsgs[topic]), len(partitions), partitions)) } } } } this.Ui.Output(fmt.Sprintf("%17s host:%d, topic:%d, partition:%d, instance:%d", "-TOTAL-", hostN, topicN, partitionN, portN)) }
func (this *Redis) render() { if !this.batchMode { termbox.Clear(termbox.ColorDefault, termbox.ColorDefault) } this.mu.Lock() defer this.mu.Unlock() if this.topOrderAsc { sortutil.AscByField(this.topInfos, this.topOrderCols[this.topOrderColIdx]) } else { sortutil.DescByField(this.topInfos, this.topOrderCols[this.topOrderColIdx]) } sortCols := make([]string, len(this.topOrderCols)) copy(sortCols, this.topOrderCols) for i, col := range sortCols { if col == this.selectedCol() { if this.topOrderAsc { sortCols[i] += " >" } else { sortCols[i] += " <" } } } var ( lines = []string{fmt.Sprintf("Host|Port|%s", strings.Join(sortCols, "|"))} sumDbsize, sumConns, sumOps, sumMem, sumRx, sumTx, sumMaxMem int64 ) for i := 0; i < len(this.topInfos); i++ { info := this.topInfos[i] sumDbsize += info.dbsize sumConns += info.conns sumOps += info.ops sumMem += info.mem sumMaxMem += info.maxmem sumRx += info.rx * 1024 / 8 sumTx += info.tx * 1024 / 8 if info.ops >= this.beep { this.warnPorts[strconv.Itoa(info.port)] = struct{}{} } this.maxDbSize = max(this.maxDbSize, info.dbsize) this.maxConns = max(this.maxConns, info.conns) this.maxOps = max(this.maxOps, info.ops) this.maxMem = max(this.maxMem, info.mem) this.maxMaxMem = max(this.maxMaxMem, info.maxmem) this.maxRx = max(this.maxRx, info.rx*1024/8) this.maxTx = max(this.maxTx, info.tx*1024/8) if info.memp > this.maxMemp { this.maxMemp = info.memp } if info.trp > this.maxTrp { this.maxTrp = info.trp } if i >= min(this.rows, len(this.topInfos)) { continue } l := fmt.Sprintf("%s|%d|%s|%s|%s|%s|%s|%6.1f|%s|%s|%8.1f", info.host, info.port, gofmt.Comma(info.dbsize), gofmt.Comma(info.conns), gofmt.Comma(info.ops), gofmt.ByteSize(info.mem), gofmt.ByteSize(info.maxmem), 100.*info.memp, gofmt.ByteSize(info.rx*1024/8), gofmt.ByteSize(info.tx*1024/8), info.trp) if this.beep > 0 { var val int64 switch this.selectedCol() { case "conns": val = info.conns case "rx": val = info.rx case "tx": val = info.tx case "dbsize": val = info.dbsize case "ops": val = info.ops case "mem": val = info.mem case "maxm": val = info.maxmem } if val > this.beep { //l += "\a" } } lines = append(lines, l) } lines = append(lines, fmt.Sprintf("-MAX-|%d|%s|%s|%s|%s|%s|%6.1f|%s|%s|%8.1f", len(this.topInfos), gofmt.Comma(this.maxDbSize), gofmt.Comma(this.maxConns), gofmt.Comma(this.maxOps), gofmt.ByteSize(this.maxMem), gofmt.ByteSize(this.maxMaxMem), 100.*this.maxMemp, gofmt.ByteSize(this.maxRx), gofmt.ByteSize(this.maxTx), this.maxTrp)) lines = append(lines, fmt.Sprintf("-TOTAL-|%d|%s|%s|%s|%s|%s|%6.1f|%s|%s|%8.1f", len(this.topInfos), gofmt.Comma(sumDbsize), gofmt.Comma(sumConns), gofmt.Comma(sumOps), gofmt.ByteSize(sumMem), gofmt.ByteSize(sumMaxMem), 100.*float64(sumMem)/float64(sumMaxMem), gofmt.ByteSize(sumRx), gofmt.ByteSize(sumTx), float64(sumTx)/float64(sumRx))) for row, line := range lines { if row == 0 { // header this.drawRow(line, row, termbox.ColorDefault, termbox.ColorBlue) } else if row == len(lines)-2 { // max line this.drawRow(line, row, termbox.ColorMagenta, termbox.ColorDefault) } else if row == len(lines)-1 { // total line this.drawRow(line, row, termbox.ColorYellow, termbox.ColorDefault) } else { tuples := strings.Split(line, "|") if _, present := this.selectedPorts[tuples[1]]; present { this.drawRow(line, row, termbox.ColorBlack, termbox.ColorCyan) } else if _, present := this.warnPorts[tuples[1]]; !present { this.drawRow(line, row, termbox.ColorDefault, termbox.ColorDefault) } else { this.drawRow(line, row, termbox.ColorDefault, termbox.ColorRed) } } } if !this.batchMode { termbox.Flush() } }
func (this *Lags) printConsumersLag(zkcluster *zk.ZkCluster) { // sort by group name consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern) sortedGroups := make([]string, 0, len(consumersByGroup)) for group, _ := range consumersByGroup { sortedGroups = append(sortedGroups, group) } sort.Strings(sortedGroups) for _, group := range sortedGroups { lines := make([]string, 0, 100) sortedTopicAndPartitionIds := make([]string, 0) consumers := make(map[string]zk.ConsumerMeta) for _, t := range consumersByGroup[group] { key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId) sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key) consumers[key] = t } sort.Strings(sortedTopicAndPartitionIds) for _, topicAndPartitionId := range sortedTopicAndPartitionIds { consumer := consumers[topicAndPartitionId] if !patternMatched(consumer.Topic, this.topicPattern) { continue } var ( lagOutput string symbol string ) if consumer.Lag > int64(this.lagThreshold) { lagOutput = color.Red("%15s", gofmt.Comma(consumer.Lag)) if consumer.Online { symbol = color.Yellow("⚠︎︎") } else { symbol = color.Yellow("◎") } } else { lagOutput = color.Blue("%15s", gofmt.Comma(consumer.Lag)) if consumer.Online { symbol = color.Green("◉") } else { symbol = color.Yellow("◎") } } if consumer.Online { if this.problematicMode && consumer.Lag <= int64(this.lagThreshold) { continue } var ( host string uptime string ) if consumer.ConsumerZnode == nil { host = "unrecognized" uptime = "-" } else { host = color.Green("%s", consumer.ConsumerZnode.Host()) if time.Since(consumer.ConsumerZnode.Uptime()) < time.Hour { uptime = color.Magenta(gofmt.PrettySince(consumer.ConsumerZnode.Uptime())) } else { uptime = gofmt.PrettySince(consumer.ConsumerZnode.Uptime()) } } lines = append(lines, fmt.Sprintf("\t%s %35s/%-2s %12s -> %-15s %s %-10s %s %s", symbol, consumer.Topic, consumer.PartitionId, gofmt.Comma(consumer.ProducerOffset), gofmt.Comma(consumer.ConsumerOffset), lagOutput, gofmt.PrettySince(consumer.Mtime.Time()), host, uptime)) } else if !this.onlineOnly { lines = append(lines, fmt.Sprintf("\t%s %35s/%-2s %12s -> %-12s %s %s", symbol, consumer.Topic, consumer.PartitionId, gofmt.Comma(consumer.ProducerOffset), gofmt.Comma(consumer.ConsumerOffset), lagOutput, gofmt.PrettySince(consumer.Mtime.Time()))) } } if len(lines) > 0 { this.Ui.Output(strings.Repeat(" ", 4) + group) for _, l := range lines { this.Ui.Output(l) } } } }
func (this *Lags) printConsumersLagTable(zkcluster *zk.ZkCluster) { lines := make([]string, 0) header := "ConsumerGroup|Topic/Partition|Produced|Consumed|Lag|Committed|Uptime" lines = append(lines, header) // sort by group name consumersByGroup := zkcluster.ConsumersByGroup(this.groupPattern) sortedGroups := make([]string, 0, len(consumersByGroup)) for group, _ := range consumersByGroup { sortedGroups = append(sortedGroups, group) } sort.Strings(sortedGroups) for _, group := range sortedGroups { if !patternMatched(group, this.groupPattern) { continue } sortedTopicAndPartitionIds := make([]string, 0, len(consumersByGroup[group])) consumers := make(map[string]zk.ConsumerMeta) for _, t := range consumersByGroup[group] { key := fmt.Sprintf("%s:%s", t.Topic, t.PartitionId) sortedTopicAndPartitionIds = append(sortedTopicAndPartitionIds, key) consumers[key] = t } sort.Strings(sortedTopicAndPartitionIds) for _, topicAndPartitionId := range sortedTopicAndPartitionIds { consumer := consumers[topicAndPartitionId] if !patternMatched(consumer.Topic, this.topicPattern) { continue } if !consumer.Online { continue } if this.problematicMode && consumer.Lag <= int64(this.lagThreshold) { continue } if consumer.ConsumerZnode == nil { this.Ui.Warn(fmt.Sprintf("%+v has no znode", consumer)) continue } lines = append(lines, fmt.Sprintf("%s|%s/%s|%s|%s|%s|%s|%s", group, consumer.Topic, consumer.PartitionId, gofmt.Comma(consumer.ProducerOffset), gofmt.Comma(consumer.ConsumerOffset), gofmt.Comma(consumer.Lag), gofmt.PrettySince(consumer.Mtime.Time()), gofmt.PrettySince(consumer.ConsumerZnode.Uptime()))) } } if len(lines) > 1 { this.Ui.Info(fmt.Sprintf("%s ▾", zkcluster.Name())) this.Ui.Output(columnize.SimpleFormat(lines)) } }
func (this *Topics) Run(args []string) (exitCode int) { var ( zone string cluster string addTopic string delTopic string killTopic, restoreTopic string replicas int partitions int retentionInMinute int resetConf bool debug bool summaryMode bool configged bool ) cmdFlags := flag.NewFlagSet("brokers", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "") cmdFlags.StringVar(&this.topicPattern, "t", "", "") cmdFlags.StringVar(&cluster, "c", "", "") cmdFlags.BoolVar(&this.verbose, "l", false, "") cmdFlags.BoolVar(&this.ipInNumber, "n", false, "") cmdFlags.StringVar(&addTopic, "add", "", "") cmdFlags.BoolVar(&summaryMode, "sum", false, "") cmdFlags.StringVar(&killTopic, "kill", "", "") cmdFlags.StringVar(&restoreTopic, "restore", "", "") cmdFlags.BoolVar(&this.plainMode, "plain", false, "") cmdFlags.StringVar(&delTopic, "del", "", "") cmdFlags.IntVar(&partitions, "partitions", 1, "") cmdFlags.DurationVar(&this.since, "since", 0, "") cmdFlags.BoolVar(&configged, "cf", false, "") cmdFlags.BoolVar(&debug, "debug", false, "") cmdFlags.BoolVar(&resetConf, "cfreset", false, "") cmdFlags.Int64Var(&this.count, "count", 0, "") cmdFlags.IntVar(&retentionInMinute, "retention", -1, "") cmdFlags.IntVar(&replicas, "replicas", 2, "") if err := cmdFlags.Parse(args); err != nil { return 1 } if validateArgs(this, this.Ui). on("-add", "-c"). on("-del", "-c"). on("-retention", "-c", "-t"). on("-cfreset", "-c", "-t"). requireAdminRights("-add", "-del", "-retention"). invalid(args) { return 2 } if this.count > 0 { this.verbose = true } if debug { sarama.Logger = log.New(os.Stderr, color.Magenta("[sarama]"), log.LstdFlags) } if addTopic != "" { zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) zkcluster := zkzone.NewCluster(cluster) swallow(this.addTopic(zkcluster, addTopic, replicas, partitions)) return } else if delTopic != "" { zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) zkcluster := zkzone.NewCluster(cluster) swallow(this.delTopic(zkcluster, delTopic)) return } ensureZoneValid(zone) zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) if retentionInMinute > 0 { zkcluster := zkzone.NewCluster(cluster) this.configTopic(zkcluster, this.topicPattern, retentionInMinute) return } if resetConf { zkcluster := zkzone.NewCluster(cluster) this.resetTopicConfig(zkcluster, this.topicPattern) configged = true // after reset, display most recent znode info } if configged { // output header this.Ui.Output(fmt.Sprintf("%25s %-40s %15s", "cluster", "topic", "mtime")) this.Ui.Output(fmt.Sprintf("%25s %40s %15s", strings.Repeat("-", 25), strings.Repeat("-", 40), strings.Repeat("-", 15))) displayTopicConfigs := func(zkcluster *zk.ZkCluster) { // sort by topic name configs := zkcluster.ConfiggedTopics() sortedTopics := make([]string, 0, len(configs)) for topic, _ := range configs { sortedTopics = append(sortedTopics, topic) } sort.Strings(sortedTopics) for _, topic := range sortedTopics { if !patternMatched(topic, this.topicPattern) { continue } configInfo := configs[topic] this.Ui.Output(fmt.Sprintf("%25s %40s %15s %s", zkcluster.Name(), topic, gofmt.PrettySince(configInfo.Mtime), configInfo.Config)) } } if cluster == "" { zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { displayTopicConfigs(zkcluster) }) } else { zkcluster := zkzone.NewCluster(cluster) displayTopicConfigs(zkcluster) } return } if summaryMode { this.printSummary(zkzone, cluster) return } if !this.verbose { // output header this.Ui.Output(fmt.Sprintf("%30s %-50s", "cluster", "topic")) this.Ui.Output(fmt.Sprintf("%30s %50s", strings.Repeat("-", 30), strings.Repeat("-", 50))) } if cluster != "" { zkcluster := zkzone.NewCluster(cluster) this.displayTopicsOfCluster(zkcluster) this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Topics-", this.topicN)) this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Partitions-", this.partitionN)) if this.verbose { if this.plainMode { fmt.Printf("%25s %s\n", "-FLAT Messages-", gofmt.Comma(this.totalMsgs)) fmt.Printf("%25s %s\n", "-CUM Messages-", gofmt.Comma(this.totalOffsets)) } else { this.Ui.Output(fmt.Sprintf("%25s %s", "-FLAT Messages-", gofmt.Comma(this.totalMsgs))) this.Ui.Output(fmt.Sprintf("%25s %s", "-CUM Messages-", gofmt.Comma(this.totalOffsets))) } } return } // all clusters zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { this.displayTopicsOfCluster(zkcluster) }) this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Topics-", this.topicN)) this.Ui.Output(fmt.Sprintf("%25s %d", "-TOTAL Partitions-", this.partitionN)) if this.verbose { if this.plainMode { fmt.Printf("%25s %s\n", "-FLAT Messages-", gofmt.Comma(this.totalMsgs)) fmt.Printf("%25s %s\n", "-CUM Messages-", gofmt.Comma(this.totalOffsets)) } else { this.Ui.Output(fmt.Sprintf("%25s %s", "-FLAT Messages-", gofmt.Comma(this.totalMsgs))) this.Ui.Output(fmt.Sprintf("%25s %s", "-CUM Messages-", gofmt.Comma(this.totalOffsets))) } } return }
func (this *Mirror) pump(sub *consumergroup.ConsumerGroup, pub sarama.AsyncProducer, stop, stopped chan struct{}) { defer func() { log.Trace("closing sub, commit offsets...") sub.Close() stopped <- struct{}{} // notify others I'm done }() active := true backoff := time.Second * 2 idle := time.Second * 10 for { select { case <-this.quit: log.Trace("got signal quit") return case <-stop: // yes sir! log.Trace("got signal stop") return case <-time.After(idle): active = false log.Info("idle 10s waiting for new message") case msg, ok := <-sub.Messages(): if !ok { log.Warn("sub encounters end of message stream") return } if !active || this.Debug { log.Info("<-[#%d] T:%s M:%s", this.transferN, msg.Topic, string(msg.Value)) } active = true pub.Input() <- &sarama.ProducerMessage{ Topic: msg.Topic, Key: sarama.ByteEncoder(msg.Key), Value: sarama.ByteEncoder(msg.Value), } if this.AutoCommit { sub.CommitUpto(msg) } // rate limit, never overflood the limited bandwidth between IDCs // FIXME when compressed, the bandwidth calculation is wrong bytesN := len(msg.Topic) + len(msg.Key) + len(msg.Value) + 20 // payload overhead if this.bandwidthRateLimiter != nil && !this.bandwidthRateLimiter.Pour(bytesN) { log.Warn("%s -> bandwidth reached, backoff %s", gofmt.ByteSize(this.transferBytes), backoff) time.Sleep(backoff) } this.transferBytes += int64(bytesN) this.transferN++ if this.transferN%this.ProgressStep == 0 { log.Trace("%s %s %s", gofmt.Comma(this.transferN), gofmt.ByteSize(this.transferBytes), msg.Topic) } case err := <-sub.Errors(): log.Error("quitting pump %v", err) return } } }
func (this *Peek) Run(args []string) (exitCode int) { var ( cluster string zone string topicPattern string partitionId int wait time.Duration silence bool ) cmdFlags := flag.NewFlagSet("peek", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "") cmdFlags.StringVar(&cluster, "c", "", "") cmdFlags.StringVar(&topicPattern, "t", "", "") cmdFlags.IntVar(&partitionId, "p", 0, "") cmdFlags.BoolVar(&this.colorize, "color", true, "") cmdFlags.Int64Var(&this.lastN, "last", -1, "") cmdFlags.BoolVar(&this.pretty, "pretty", false, "") cmdFlags.IntVar(&this.limit, "n", -1, "") cmdFlags.StringVar(&this.column, "col", "", "") // TODO support multiple columns cmdFlags.BoolVar(&this.beep, "beep", false, "") cmdFlags.Int64Var(&this.offset, "offset", sarama.OffsetNewest, "") cmdFlags.BoolVar(&silence, "s", false, "") cmdFlags.DurationVar(&wait, "d", time.Hour, "") cmdFlags.BoolVar(&this.bodyOnly, "body", false, "") if err := cmdFlags.Parse(args); err != nil { return 1 } if this.pretty { this.bodyOnly = true } this.quit = make(chan struct{}) if silence { stats := newPeekStats() go stats.start() } zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) msgChan := make(chan *sarama.ConsumerMessage, 20000) // msg aggerator channel if cluster == "" { zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { this.consumeCluster(zkcluster, topicPattern, partitionId, msgChan) }) } else { zkcluster := zkzone.NewCluster(cluster) this.consumeCluster(zkcluster, topicPattern, partitionId, msgChan) } signal.RegisterHandler(func(sig os.Signal) { log.Printf("received signal: %s", strings.ToUpper(sig.String())) log.Println("quiting...") this.once.Do(func() { close(this.quit) }) }, syscall.SIGINT, syscall.SIGTERM) var ( startAt = time.Now() msg *sarama.ConsumerMessage total int bytesN int64 ) var ( j map[string]interface{} prettyJSON bytes.Buffer ) LOOP: for { if time.Since(startAt) >= wait { this.Ui.Output(fmt.Sprintf("Total: %s msgs, %s, elapsed: %s", gofmt.Comma(int64(total)), gofmt.ByteSize(bytesN), time.Since(startAt))) elapsed := time.Since(startAt).Seconds() if elapsed > 1. { this.Ui.Output(fmt.Sprintf("Speed: %d/s", total/int(elapsed))) if total > 0 { this.Ui.Output(fmt.Sprintf("Size : %s/msg", gofmt.ByteSize(bytesN/int64(total)))) } } return } select { case <-this.quit: this.Ui.Output(fmt.Sprintf("Total: %s msgs, %s, elapsed: %s", gofmt.Comma(int64(total)), gofmt.ByteSize(bytesN), time.Since(startAt))) elapsed := time.Since(startAt).Seconds() if elapsed > 1. { this.Ui.Output(fmt.Sprintf("Speed: %d/s", total/int(elapsed))) if total > 0 { this.Ui.Output(fmt.Sprintf("Size : %s/msg", gofmt.ByteSize(bytesN/int64(total)))) } } return case <-time.After(time.Second): continue case msg = <-msgChan: if silence { stats.MsgCountPerSecond.Mark(1) stats.MsgBytesPerSecond.Mark(int64(len(msg.Value))) } else { var outmsg string if this.column != "" { if err := json.Unmarshal(msg.Value, &j); err != nil { this.Ui.Error(err.Error()) } else { var colVal string switch t := j[this.column].(type) { case string: colVal = t case float64: colVal = fmt.Sprintf("%.0f", t) case int: colVal = fmt.Sprintf("%d", t) } if this.bodyOnly { if this.pretty { if err = json.Indent(&prettyJSON, []byte(colVal), "", " "); err != nil { fmt.Println(err.Error()) } else { outmsg = string(prettyJSON.Bytes()) } } else { outmsg = colVal } } else if this.colorize { outmsg = fmt.Sprintf("%s/%d %s k:%s v:%s", color.Green(msg.Topic), msg.Partition, gofmt.Comma(msg.Offset), string(msg.Key), colVal) } else { // colored UI will have invisible chars output outmsg = fmt.Sprintf("%s/%d %s k:%s v:%s", msg.Topic, msg.Partition, gofmt.Comma(msg.Offset), string(msg.Key), colVal) } } } else { if this.bodyOnly { if this.pretty { json.Indent(&prettyJSON, msg.Value, "", " ") outmsg = string(prettyJSON.Bytes()) } else { outmsg = string(msg.Value) } } else if this.colorize { outmsg = fmt.Sprintf("%s/%d %s k:%s, v:%s", color.Green(msg.Topic), msg.Partition, gofmt.Comma(msg.Offset), string(msg.Key), string(msg.Value)) } else { // colored UI will have invisible chars output outmsg = fmt.Sprintf("%s/%d %s k:%s, v:%s", msg.Topic, msg.Partition, gofmt.Comma(msg.Offset), string(msg.Key), string(msg.Value)) } } if outmsg != "" { if this.beep { outmsg += "\a" } this.Ui.Output(outmsg) } } total++ bytesN += int64(len(msg.Value)) if this.limit > 0 && total >= this.limit { break LOOP } if this.lastN > 0 && total >= int(this.lastN) { break LOOP } } } return }
func (e *EngineConfig) ServeForever() { var ( outputsWg = new(sync.WaitGroup) filtersWg = new(sync.WaitGroup) inputsWg = new(sync.WaitGroup) globals = Globals() err error ) // setup signal handler first to avoid race condition // if Input terminates very soon, global.Shutdown will // not be able to trap it globals.sigChan = make(chan os.Signal) signal.Notify(globals.sigChan, syscall.SIGINT, syscall.SIGHUP, syscall.SIGUSR2, syscall.SIGUSR1) e.launchHttpServ() if globals.Verbose { globals.Println("Launching Output(s)...") } for _, runner := range e.OutputRunners { outputsWg.Add(1) if err = runner.start(e, outputsWg); err != nil { panic(err) } } if globals.Verbose { globals.Println("Launching Filter(s)...") } for _, runner := range e.FilterRunners { filtersWg.Add(1) if err = runner.start(e, filtersWg); err != nil { panic(err) } } // setup the diagnostic trackers inputPackTracker := NewDiagnosticTracker("inputPackTracker") e.diagnosticTrackers[inputPackTracker.PoolName] = inputPackTracker filterPackTracker := NewDiagnosticTracker("filterPackTracker") e.diagnosticTrackers[filterPackTracker.PoolName] = filterPackTracker if globals.Verbose { globals.Printf("Initializing PipelinePack pools with size=%d\n", globals.RecyclePoolSize) } for i := 0; i < globals.RecyclePoolSize; i++ { inputPack := NewPipelinePack(e.inputRecycleChan) inputPackTracker.AddPack(inputPack) e.inputRecycleChan <- inputPack filterPack := NewPipelinePack(e.filterRecycleChan) filterPackTracker.AddPack(filterPack) e.filterRecycleChan <- filterPack } go inputPackTracker.Run(e.Int("diagnostic_interval", 20)) go filterPackTracker.Run(e.Int("diagnostic_interval", 20)) // check if we have enough recycle pool reservation go func() { t := time.NewTicker(time.Second * time.Duration(globals.TickerLength)) defer t.Stop() var inputPoolSize, filterPoolSize int for _ = range t.C { inputPoolSize = len(e.inputRecycleChan) filterPoolSize = len(e.filterRecycleChan) if globals.Verbose || inputPoolSize == 0 || filterPoolSize == 0 { globals.Printf("Recycle poolSize: [input]%d [filter]%d", inputPoolSize, filterPoolSize) } } }() go e.router.Start() for _, project := range e.projects { project.Start() } if globals.Verbose { globals.Println("Launching Input(s)...") } for _, runner := range e.InputRunners { inputsWg.Add(1) if err = runner.start(e, inputsWg); err != nil { inputsWg.Done() panic(err) } } globals.Println("Engine mainloop, waiting for signals...") go runShutdownWatchdog(e) for !globals.Stopping { select { case sig := <-globals.sigChan: globals.Printf("Got signal %s\n", sig.String()) switch sig { case syscall.SIGHUP: globals.Println("Reloading...") observer.Publish(RELOAD, nil) case syscall.SIGINT: globals.Println("Engine shutdown...") globals.Stopping = true case syscall.SIGUSR1: observer.Publish(SIGUSR1, nil) case syscall.SIGUSR2: observer.Publish(SIGUSR2, nil) } } } // cleanup after shutdown inputPackTracker.Stop() filterPackTracker.Stop() e.Lock() for _, runner := range e.InputRunners { if runner == nil { // this Input plugin already exit continue } if globals.Verbose { globals.Printf("Stop message sent to '%s'", runner.Name()) } runner.Input().Stop() } e.Unlock() inputsWg.Wait() // wait for all inputs done if globals.Verbose { globals.Println("All Inputs terminated") } // ok, now we are sure no more inputs, but in route.inChan there // still may be filter injected packs and output not consumed packs // we must wait for all the packs to be consumed before shutdown for _, runner := range e.FilterRunners { if globals.Verbose { globals.Printf("Stop message sent to '%s'", runner.Name()) } e.router.removeFilterMatcher <- runner.Matcher() } filtersWg.Wait() if globals.Verbose { globals.Println("All Filters terminated") } for _, runner := range e.OutputRunners { if globals.Verbose { globals.Printf("Stop message sent to '%s'", runner.Name()) } e.router.removeOutputMatcher <- runner.Matcher() } outputsWg.Wait() if globals.Verbose { globals.Println("All Outputs terminated") } //close(e.router.inChan) e.stopHttpServ() for _, project := range e.projects { project.Stop() } globals.Printf("Shutdown with input:%s, dispatched:%s", gofmt.Comma(e.router.stats.TotalInputMsgN), gofmt.Comma(e.router.stats.TotalProcessedMsgN)) }
func (this *Topics) displayTopicsOfCluster(zkcluster *zk.ZkCluster) { echoBuffer := func(lines []string) { for _, l := range lines { this.Ui.Output(l) } } linesInTopicMode := make([]string, 0) if this.verbose { linesInTopicMode = this.echoOrBuffer(zkcluster.Name(), linesInTopicMode) } // get all alive brokers within this cluster brokers := zkcluster.Brokers() if len(brokers) == 0 { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s", " ", color.Red("%s empty brokers", zkcluster.Name())), linesInTopicMode) echoBuffer(linesInTopicMode) return } if this.verbose { sortedBrokerIds := make([]string, 0, len(brokers)) for brokerId, _ := range brokers { sortedBrokerIds = append(sortedBrokerIds, brokerId) } sort.Strings(sortedBrokerIds) for _, brokerId := range sortedBrokerIds { if this.ipInNumber { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ", color.Green(brokerId), brokers[brokerId]), linesInTopicMode) } else { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%4s%s %s", " ", color.Green(brokerId), brokers[brokerId].NamedString()), linesInTopicMode) } } } kfk, err := sarama.NewClient(zkcluster.BrokerList(), saramaConfig()) if err != nil { if this.verbose { linesInTopicMode = this.echoOrBuffer(color.Yellow("%5s%+v %s", " ", zkcluster.BrokerList(), err.Error()), linesInTopicMode) } return } defer kfk.Close() topics, err := kfk.Topics() swallow(err) if len(topics) == 0 { if this.topicPattern == "" && this.verbose { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%5s%s", " ", color.Magenta("no topics")), linesInTopicMode) echoBuffer(linesInTopicMode) } return } sortedTopics := make([]string, 0, len(topics)) for _, t := range topics { sortedTopics = append(sortedTopics, t) } sort.Strings(sortedTopics) topicsCtime := zkcluster.TopicsCtime() hasTopicMatched := false for _, topic := range sortedTopics { if !patternMatched(topic, this.topicPattern) { continue } if this.since > 0 && time.Since(topicsCtime[topic]) > this.since { continue } this.topicN++ hasTopicMatched = true if this.verbose { linesInTopicMode = this.echoOrBuffer(strings.Repeat(" ", 4)+color.Cyan(topic), linesInTopicMode) } // get partitions and check if some dead alivePartitions, err := kfk.WritablePartitions(topic) swallow(err) partions, err := kfk.Partitions(topic) swallow(err) if len(alivePartitions) != len(partions) { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %s P: %s/%+v", zkcluster.Name(), color.Cyan("%-50s", topic), color.Red("partial dead"), color.Green("%+v", alivePartitions), partions), linesInTopicMode) } replicas, err := kfk.Replicas(topic, partions[0]) if err != nil { this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partions[0], err)) } this.partitionN += len(partions) if !this.verbose { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%30s %s %3dP %dR %s", zkcluster.Name(), color.Cyan("%-50s", topic), len(partions), len(replicas), gofmt.PrettySince(topicsCtime[topic])), linesInTopicMode) continue } for _, partitionID := range alivePartitions { leader, err := kfk.Leader(topic, partitionID) swallow(err) replicas, err := kfk.Replicas(topic, partitionID) if err != nil { this.Ui.Error(fmt.Sprintf("%s/%d %v", topic, partitionID, err)) } isr, isrMtime, partitionCtime := zkcluster.Isr(topic, partitionID) isrMtimeSince := gofmt.PrettySince(isrMtime) if time.Since(isrMtime).Hours() < 24 { // ever out of sync last 24h isrMtimeSince = color.Magenta(isrMtimeSince) } underReplicated := false if len(isr) != len(replicas) { underReplicated = true } latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest) swallow(err) oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest) swallow(err) if this.count > 0 && (latestOffset-oldestOffset) < this.count { continue } this.totalMsgs += latestOffset - oldestOffset this.totalOffsets += latestOffset if !underReplicated { linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%+v Offset:%16s - %-16s Num:%-15s %s-%s", partitionID, color.Green("%d", leader.ID()), replicas, isr, gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset), gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode) } else { // use red for alert linesInTopicMode = this.echoOrBuffer(fmt.Sprintf("%8d Leader:%s Replicas:%+v Isr:%s Offset:%16s - %-16s Num:%-15s %s-%s", partitionID, color.Green("%d", leader.ID()), replicas, color.Red("%+v", isr), gofmt.Comma(oldestOffset), gofmt.Comma(latestOffset), gofmt.Comma(latestOffset-oldestOffset), gofmt.PrettySince(partitionCtime), isrMtimeSince), linesInTopicMode) } } } if this.topicPattern != "" { if hasTopicMatched { echoBuffer(linesInTopicMode) } } else { echoBuffer(linesInTopicMode) } }
func (this *Lags) Run(args []string) (exitCode int) { const secondsInMinute = 60 var ( cluster string zone string ) cmdFlags := flag.NewFlagSet("lags", flag.ContinueOnError) cmdFlags.Usage = func() { this.Ui.Output(this.Help()) } cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "") cmdFlags.StringVar(&cluster, "c", "", "") cmdFlags.BoolVar(&this.onlineOnly, "l", false, "") cmdFlags.BoolVar(&this.problematicMode, "p", false, "") cmdFlags.StringVar(&this.groupPattern, "g", "", "") cmdFlags.StringVar(&this.topicPattern, "t", "", "") cmdFlags.BoolVar(&this.tableFmt, "table", false, "") cmdFlags.BoolVar(&this.watchMode, "w", false, "") cmdFlags.IntVar(&this.lagThreshold, "lag", 5000, "") if err := cmdFlags.Parse(args); err != nil { return 1 } if this.watchMode { refreshScreen() } if this.problematicMode { this.onlineOnly = true } zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone))) bar := progress.New(secondsInMinute) if cluster == "" { for { zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) { if this.tableFmt { this.printConsumersLagTable(zkcluster) } else { this.printConsumersLag(zkcluster) } }) if this.watchMode { for i := 1; i <= secondsInMinute; i++ { bar.ShowProgress(i) time.Sleep(time.Second) } } else { break } printSwallowedErrors(this.Ui, zkzone) } this.Ui.Output(fmt.Sprintf("Lag totals: %s", gofmt.Comma(this.lagTotal))) return } // display a single cluster for { zkcluster := zkzone.NewCluster(cluster) // panic if invalid cluster if this.tableFmt { this.printConsumersLagTable(zkcluster) } else { this.printConsumersLag(zkcluster) } if this.watchMode { for i := 1; i <= secondsInMinute; i++ { bar.ShowProgress(i) time.Sleep(time.Second) } //refreshScreen() } else { break } printSwallowedErrors(this.Ui, zkzone) } this.Ui.Output(fmt.Sprintf("Lag totals: %s", gofmt.Comma(this.lagTotal))) return }
func (this *Mirror) runMirror(c1, c2 *zk.ZkCluster, limit int64) { this.startedAt = time.Now() log.Info("start [%s/%s] -> [%s/%s] with bandwidth %sbps", c1.ZkZone().Name(), c1.Name(), c2.ZkZone().Name(), c2.Name(), gofmt.Comma(limit*8)) pub, err := this.makePub(c2) if err != nil { panic(err) } log.Trace("pub[%s/%s] created", c2.ZkZone().Name(), c2.Name()) go func(pub sarama.AsyncProducer, c *zk.ZkCluster) { for { select { case <-this.quit: return case err := <-pub.Errors(): // messages will only be returned here after all retry attempts are exhausted. // // e,g // Failed to produce message to topic xx: write tcp src->kfk: i/o timeout // kafka: broker not connected log.Error("pub[%s/%s] %v", c.ZkZone().Name(), c.Name(), err) } } }(pub, c2) group := this.groupName(c1, c2) ever := true round := 0 for ever { round++ topics, topicsChanges, err := c1.WatchTopics() if err != nil { log.Error("#%d [%s/%s]watch topics: %v", round, c1.ZkZone().Name(), c1.Name(), err) time.Sleep(time.Second * 10) continue } topics = this.realTopics(topics) sub, err := this.makeSub(c1, group, topics) if err != nil { log.Error("#%d [%s/%s] %v", round, c1.ZkZone().Name(), c1.Name(), err) time.Sleep(time.Second * 10) continue } log.Info("#%d starting pump [%s/%s] -> [%s/%s] %d topics with group %s for %+v", round, c1.ZkZone().Name(), c1.Name(), c2.ZkZone().Name(), c2.Name(), len(topics), group, topics) pumpStopper := make(chan struct{}) pumpStopped := make(chan struct{}) go this.pump(sub, pub, pumpStopper, pumpStopped) select { case <-topicsChanges: // TODO log the diff the topics log.Warn("#%d [%s/%s] topics changed, stopping pump...", round, c1.Name(), c2.Name()) pumpStopper <- struct{}{} // stop pump <-pumpStopped // await pump cleanup case <-this.quit: log.Info("#%d awaiting pump cleanup...", round) <-pumpStopped ever = false case <-pumpStopped: // pump encounters problems, just retry } } log.Info("total transferred: %s %smsgs", gofmt.ByteSize(this.transferBytes), gofmt.Comma(this.transferN)) log.Info("closing pub...") pub.Close() }