func (this *WatchConsumers) runSubQpsTimer() {
this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
consumerGroups := zkcluster.ConsumerGroups()
for group, _ := range consumerGroups {
offsetMap := zkcluster.ConsumerOffsetsOfGroup(group)
for topic, m := range offsetMap {
offsetOfGroupOnTopic := int64(0)
for _, offset := range m {
offsetOfGroupOnTopic += offset
}
// cluster, topic, group, offset
tag := telemetry.Tag(zkcluster.Name(), strings.Replace(topic, ".", "_", -1), strings.Replace(group, ".", "_", -1))
if _, present := this.consumerQps[tag]; !present {
this.consumerQps[tag] = metrics.NewRegisteredMeter(tag+"consumer.qps", nil)
}
lastOffset := this.lastOffsets[tag]
if lastOffset == 0 {
// first run
this.lastOffsets[tag] = offsetOfGroupOnTopic
} else {
delta := offsetOfGroupOnTopic - lastOffset
if delta >= 0 {
this.consumerQps[tag].Mark(delta)
this.lastOffsets[tag] = offsetOfGroupOnTopic
} else {
log.Warn("cluster[%s] topic[%s] group[%s] offset rewinds: %d %d",
zkcluster.Name(), topic, group, offsetOfGroupOnTopic, lastOffset)
}
}
}
}
})
}
func (this *WatchSlowlog) Run() {
defer this.Wg.Done()
ticker := time.NewTicker(this.Tick)
defer ticker.Stop()
this.slows = make(map[string]metrics.Gauge, 10)
for {
select {
case <-this.Stop:
log.Info("redis.slowlog stopped")
return
case <-ticker.C:
var wg sync.WaitGroup
for _, hostPort := range this.Zkzone.AllRedis() {
host, port, err := net.SplitHostPort(hostPort)
if err != nil {
log.Error("invalid redis instance: %s", hostPort)
continue
}
nport, err := strconv.Atoi(port)
if err != nil || nport < 0 {
log.Error("invalid redis instance: %s", hostPort)
continue
}
var ip string
ips, err := net.LookupIP(host) // host in ip form is also ok e,g. 10.1.1.1
if err != nil {
log.Error("redis host[%s] ip: %v", host, err)
} else if len(ips) > 0 {
ip = ips[0].String()
}
tag := telemetry.Tag(strings.Replace(host, ".", "_", -1), port, ip)
wg.Add(1)
go this.updateRedisSlowlog(&wg, host, nport, tag)
}
wg.Wait()
}
}
}
func (this *WatchRedisInfo) Run() {
defer this.Wg.Done()
ticker := time.NewTicker(this.Tick)
defer ticker.Stop()
this.instances = metrics.NewRegisteredGauge("redis.n", nil)
this.hosts = metrics.NewRegisteredGauge("redis.hosts", nil)
this.deadInstance = metrics.NewRegisteredGauge("redis.dead", nil)
this.syncPartial = metrics.NewRegisteredGauge("redis.sync.partial", nil)
this.conns = make(map[string]metrics.Gauge, 10)
this.blocked = make(map[string]metrics.Gauge, 10)
this.usedMem = make(map[string]metrics.Gauge, 10)
this.ops = make(map[string]metrics.Gauge, 10)
this.rejected = make(map[string]metrics.Gauge, 10)
this.rxKbps = make(map[string]metrics.Gauge, 10)
this.txKbps = make(map[string]metrics.Gauge, 10)
this.expiredKeys = make(map[string]metrics.Gauge, 10)
this.keys = make(map[string]metrics.Gauge, 10)
for {
select {
case <-this.Stop:
log.Info("redis.info stopped")
return
case <-ticker.C:
var (
wg sync.WaitGroup
redisN int64
hosts = make(map[string]struct{})
)
for _, hostPort := range this.Zkzone.AllRedis() {
host, port, err := net.SplitHostPort(hostPort)
if err != nil {
log.Error("invalid redis instance: %s", hostPort)
continue
}
nport, err := strconv.Atoi(port)
if err != nil || nport < 0 {
log.Error("invalid redis instance: %s", hostPort)
continue
}
redisN++
var ip string
ips, err := net.LookupIP(host) // host in ip form is also ok e,g. 10.1.1.1
if err != nil {
log.Error("redis host[%s] ip: %v", host, err)
} else if len(ips) > 0 {
ip = ips[0].String()
hosts[ip] = struct{}{}
}
tag := telemetry.Tag(strings.Replace(host, ".", "_", -1), port, ip)
if _, present := this.conns[tag]; !present {
this.mu.Lock()
this.conns[tag] = metrics.NewRegisteredGauge(tag+"redis.conns", nil) // connected_clients
this.blocked[tag] = metrics.NewRegisteredGauge(tag+"redis.blocked", nil) // blocked_clients
this.usedMem[tag] = metrics.NewRegisteredGauge(tag+"redis.mem.used", nil) // used_memory
this.ops[tag] = metrics.NewRegisteredGauge(tag+"redis.ops", nil) // instantaneous_ops_per_sec
this.rejected[tag] = metrics.NewRegisteredGauge(tag+"redis.rejected", nil) // rejected_connections
this.rxKbps[tag] = metrics.NewRegisteredGauge(tag+"redis.rx.kbps", nil) // instantaneous_input_kbps
this.txKbps[tag] = metrics.NewRegisteredGauge(tag+"redis.tx.kbps", nil) // instantaneous_output_kbps
this.expiredKeys[tag] = metrics.NewRegisteredGauge(tag+"redis.expired.keys", nil) // expired_keys
this.keys[tag] = metrics.NewRegisteredGauge(tag+"redis.keys", nil) // db0:keys=15500,expires=15500,avg_ttl=27438570
this.mu.Unlock()
}
wg.Add(1)
go this.updateRedisInfo(&wg, host, nport, tag)
}
this.instances.Update(redisN)
this.syncPartial.Update(atomic.LoadInt64(&this.syncPartialN))
this.deadInstance.Update(atomic.LoadInt64(&this.deadN))
this.hosts.Update(int64(len(hosts)))
// reset for next round
atomic.StoreInt64(&this.syncPartialN, 0)
atomic.StoreInt64(&this.deadN, 0)
wg.Wait()
}
}
}
