func (this *AccessLogger) doRotate() {
var fname string
_, err := os.Lstat(this.filename)
if err == nil {
// file exists, find a empty slot
num := 1
for ; err == nil && num <= 999; num++ {
fname = this.filename + fmt.Sprintf(".%03d", num)
_, err = os.Lstat(fname)
}
if err == nil {
log.Error("Access logger unable to rotate, 30 years passed?")
return
}
}
this.fd.Close()
this.fd = nil
// if fd does not close, after rename, fd.Write happens
// content will be written to new file
err = os.Rename(this.filename, fname)
if err != nil {
log.Error("rename %s->%s: %v", this.filename, fname)
return
}
if this.fd, err = os.OpenFile(this.filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0660); err != nil {
log.Error("open(%s): %s", this.filename, err)
}
}
func (this *mysqlStore) Start() error {
if err := this.refreshFromMysql(); err != nil {
// refuse to start if mysql conn fails
return fmt.Errorf("manager[%s]: %v", this.Name(), err)
}
go func() {
ticker := time.NewTicker(this.cf.Refresh)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if err := this.refreshFromMysql(); err != nil {
log.Error(err.Error())
} else {
log.Info("manager refreshed from mysql")
}
case <-this.refreshCh:
if err := this.refreshFromMysql(); err != nil {
log.Error(err.Error())
} else {
log.Info("manager forced to refresh from mysql")
}
case <-this.shutdownCh:
log.Info("mysql manager stopped")
return
}
}
}()
return nil
}
func (this *Client) Warmup() {
var (
sess *mgo.Session
err error
t1 = time.Now()
)
for retries := 0; retries < 3; retries++ {
for _, server := range this.selector.ServerList() {
sess, err = this.getConn(server.Uri())
if err != nil {
log.Error("Warmup %v fail: %s", server.Uri(), err)
break
} else {
this.putFreeConn(server.Uri(), sess)
}
}
if err == nil {
break
}
}
if err == nil {
log.Trace("Mongodb warmup within %s: %+v",
time.Since(t1), this.freeconns)
} else {
log.Error("Mongodb failed to warmup within %s: %s",
time.Since(t1), err)
}
}
func (this *Ping) diagnose() {
this.zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
registeredBrokers := zkcluster.RegisteredInfo().Roster
for _, broker := range registeredBrokers {
log.Debug("ping %s", broker.Addr())
kfk, err := sarama.NewClient([]string{broker.Addr()}, sarama.NewConfig())
if err != nil {
log.Error("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Red(err.Error()))
continue
}
_, err = kfk.Topics() // kafka didn't provide ping, so use Topics() as ping
if err != nil {
log.Error("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Red(err.Error()))
} else {
if !this.problematicMode {
log.Info("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Green("ok"))
}
}
kfk.Close()
}
})
}
func (q *queue) housekeeping() {
defer func() {
log.Trace("queue[%s] housekeeping done", q.ident())
q.wg.Done()
}()
log.Trace("queue[%s] start housekeeping...", q.ident())
purgeTick := time.NewTicker(q.purgeInterval)
defer purgeTick.Stop()
cursorChkpnt := time.NewTicker(time.Second)
defer cursorChkpnt.Stop()
for {
select {
case <-purgeTick.C:
if err := q.Purge(); err != nil {
log.Error("queue[%s] purge: %s", q.ident(), err)
}
case <-cursorChkpnt.C:
if err := q.cursor.dump(); err != nil {
log.Error("queue[%s] cursor checkpoint: %s", q.ident(), err)
}
case <-q.quit:
return
}
}
}
// InfluxDB starts a InfluxDB reporter which will post the metrics from the given registry at each d interval.
// CREATE RETENTION POLICY two_hours ON food_data DURATION 2h REPLICATION 1 DEFAULT
// SHOW RETENTION POLICIES ON food_data
// CREATE CONTINUOUS QUERY cq_30m ON food_data BEGIN SELECT mean(website) AS mean_website,mean(phone) AS mean_phone INTO food_data."default".downsampled_orders FROM orders GROUP BY time(30m) END
func InfluxDB(hostname string, r metrics.Registry, interval time.Duration,
url, database, username, password string, stop chan struct{}) {
u, err := uurl.Parse(url)
if err != nil {
log.Error("unable to parse InfluxDB url %s. err=%v", url, err)
return
}
rep := &reporter{
reg: r,
interval: interval,
stop: stop,
url: *u,
database: database,
hostname: hostname,
username: username,
password: password,
}
if err := rep.makeClient(); err != nil {
log.Error("unable to make InfluxDB client. err=%v", err)
return
}
rep.run()
}
func (this *Start) reloadHAproxy() (err error) {
var cmd *exec.Cmd = nil
waitStartCh := make(chan struct{})
if this.starting {
log.Info("haproxy starting")
cmd = exec.Command(this.command, "-f", configFile) // TODO use absolute path
this.starting = false
go func() {
<-waitStartCh
log.Info("haproxy started")
if err := cmd.Wait(); err != nil {
log.Error("haproxy: %v", err)
}
}()
} else {
shellScript := fmt.Sprintf("%s -f %s/%s -sf `cat %s/%s`",
this.command, this.root, configFile, this.root, haproxyPidFile)
log.Info("haproxy reloading: %s", shellScript)
cmd = exec.Command("/bin/sh", "-c", shellScript)
go func() {
<-waitStartCh
log.Info("haproxy reloaded")
if err := cmd.Wait(); err != nil {
log.Error("haproxy: %v", err)
}
}()
}
if err = cmd.Start(); err == nil {
waitStartCh <- struct{}{}
}
return err
}
func loadTemplates() {
if config.faeTemplateFile != "" {
body, err := ioutil.ReadFile(config.faeTemplateFile)
if err != nil {
log.Error("template[%s]: %s", config.faeTemplateFile, err)
} else {
faeTemplateContents = string(body)
log.Info("template[%s] loaded", config.faeTemplateFile)
}
}
if config.actorTemplateFile != "" {
body, err := ioutil.ReadFile(config.actorTemplateFile)
if err != nil {
log.Error("template[%s]: %s", config.actorTemplateFile, err)
} else {
maintainTemplateContents = string(body)
log.Info("template[%s] loaded", config.actorTemplateFile)
}
}
if config.maintainTemplateFile != "" {
body, err := ioutil.ReadFile(config.maintainTemplateFile)
if err != nil {
log.Error("template[%s]: %s", config.maintainTemplateFile, err)
} else {
maintainTemplateContents = string(body)
log.Info("template[%s] loaded", config.maintainTemplateFile)
}
}
}
func (f *File) Attr(ctx context.Context, o *fuse.Attr) error {
f.RLock()
defer f.RUnlock()
*o = f.attr
// calculate size
if !f.opened {
if err := f.dir.reconnectKafkaIfNecessary(); err != nil {
return err
}
latestOffset, err := f.dir.GetOffset(f.topic, f.partitionId, sarama.OffsetNewest)
if err != nil {
log.Error(err)
return err
}
oldestOffset, err := f.dir.GetOffset(f.topic, f.partitionId, sarama.OffsetOldest)
if err != nil {
log.Error(err)
return err
}
o.Size = uint64(latestOffset - oldestOffset)
} else {
o.Size = uint64(len(f.content))
}
log.Trace("File Attr, topic=%s, partitionId=%d, size=%d", f.topic, f.partitionId, o.Size)
return nil
}
func (this *Start) shutdown() {
// kill haproxy
log.Info("killling haproxy processes")
f, e := os.Open(haproxyPidFile)
if e != nil {
log.Error("shutdown %v", e)
return
}
reader := bufio.NewReader(f)
for {
l, e := gio.ReadLine(reader)
if e != nil {
// EOF
break
}
pid, _ := strconv.Atoi(string(l))
p := &os.Process{
Pid: pid,
}
if err := p.Kill(); err != nil {
log.Error(err)
} else {
log.Info("haproxy[%d] terminated", pid)
}
}
log.Info("removing %s", haproxyPidFile)
os.Remove(haproxyPidFile)
}
func (this *WatchSlowlog) updateRedisSlowlog(wg *sync.WaitGroup, host string, port int, tag string) {
defer wg.Done()
spec := redis.DefaultSpec().Host(host).Port(port)
client, err := redis.NewSynchClientWithSpec(spec)
if err != nil {
log.Error("redis[%s:%d]: %v", host, port, err)
return
}
defer client.Quit()
n, err := client.SlowlogLen()
if err != nil {
log.Error("redis[%s:%d]: %v", host, port, err)
return
}
if n == 0 {
return
}
this.mu.Lock()
this.slows[tag] = metrics.NewRegisteredGauge(tag+"redis.slowlog", nil)
this.slows[tag].Update(n)
this.mu.Unlock()
}
func (this *haproxyMetrics) updateMetrics(records map[string]map[string]int64) (err error) {
for svcName, svcCols := range records {
//find svc
svcMetrics, present := this.svcMetricsMap[svcName]
if !present {
log.Error("svcName[%s] not in svcMetricsMap[%#v]", svcName, this.svcMetricsMap)
err = ErrUnsupService
return err
}
//update col value
for colName, colVal := range svcCols {
//find gague
gauge, present := svcMetrics.metricsMap[colName]
if !present {
log.Error("colName[%s] not in metricsMap[%#v]", colName, svcMetrics.metricsMap)
err = ErrMetricsNotFound
return err
}
//update gauge
gauge.Update(colVal)
}
}
return
}
// CallSOS will send SOS message to the zone wide kguard leader.
func (this *ZkZone) CallSOS(caller string, msg string) {
log.Critical("SOS[%s] %s: sending...", caller, msg)
// kguard leader might float, so refresh on each SOS message
kguards, err := this.KguardInfos()
if err != nil {
log.Error("SOS[%s] %s: %v", caller, msg, err)
return
}
leader := kguards[0]
request := gorequest.New().Timeout(time.Second * 10)
res, body, errs := request.Post(fmt.Sprintf("http://%s:%d", leader.Host, telemetry.SOSPort)).
Set("User-Agent", fmt.Sprintf("sos-go-%s", gafka.BuildId)).
Set(telemetry.SOSIdentHeader, caller).
End()
if len(errs) > 0 {
log.Error("SOS[%s] %s: %+v", caller, msg, errs)
return
}
if res.StatusCode != http.StatusAccepted {
log.Error("SOS[%s] %s: HTTP %s %s", caller, msg, http.StatusText(res.StatusCode), body)
return
}
log.Info("SOS[%s] %s: sent ok", caller, msg)
}
// @rest POST /v1/jobs/:appid/:topic/:ver
func (this *manServer) createJobHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
topic := params.ByName(UrlParamTopic)
if !manager.Default.ValidateTopicName(topic) {
log.Warn("illegal topic: %s", topic)
writeBadRequest(w, "illegal topic")
return
}
realIp := getHttpRemoteIp(r)
if !this.throttleAddTopic.Pour(realIp, 1) {
writeQuotaExceeded(w)
return
}
hisAppid := params.ByName(UrlParamAppid)
appid := r.Header.Get(HttpHeaderAppid)
pubkey := r.Header.Get(HttpHeaderPubkey)
ver := params.ByName(UrlParamVersion)
if !manager.Default.AuthAdmin(appid, pubkey) {
log.Warn("suspicous create job %s(%s) {appid:%s pubkey:%s topic:%s ver:%s}",
r.RemoteAddr, realIp, appid, pubkey, topic, ver)
writeAuthFailure(w, manager.ErrAuthenticationFail)
return
}
cluster, found := manager.Default.LookupCluster(hisAppid)
if !found {
log.Error("create job %s(%s) {appid:%s topic:%s ver:%s} invalid appid",
r.RemoteAddr, realIp, hisAppid, topic, ver)
writeBadRequest(w, "invalid appid")
return
}
log.Info("create job[%s] %s(%s) {appid:%s topic:%s ver:%s}",
appid, r.RemoteAddr, realIp, hisAppid, topic, ver)
rawTopic := manager.Default.KafkaTopic(hisAppid, topic, ver)
if err := job.Default.CreateJobQueue(Options.AssignJobShardId, hisAppid, rawTopic); err != nil {
log.Error("create job[%s] %s(%s) {shard:%d appid:%s topic:%s ver:%s} %v",
appid, r.RemoteAddr, realIp, Options.AssignJobShardId, hisAppid, topic, ver, err)
writeServerError(w, err.Error())
return
}
if err := this.gw.zkzone.CreateJobQueue(rawTopic, cluster); err != nil {
log.Error("app[%s] %s(%s) create job: {shard:%d appid:%s topic:%s ver:%s} %v",
appid, r.RemoteAddr, realIp, Options.AssignJobShardId, hisAppid, topic, ver, err)
writeServerError(w, err.Error())
return
}
w.WriteHeader(http.StatusCreated)
w.Write(ResponseOk)
}
func (r *reporter) run() {
intervalTicker := time.Tick(r.interval)
//pingTicker := time.Tick(time.Second * 5)
pingTicker := time.Tick(r.interval / 2)
for {
select {
// TODO on shutdown, flush all metrics
case <-r.stop:
return
case <-intervalTicker:
if err := r.send(); err != nil {
log.Error("unable to send metrics to InfluxDB. err=%v", err)
}
case <-pingTicker:
_, _, err := r.client.Ping()
if err != nil {
log.Error("got error while sending a ping to InfluxDB, trying to recreate client. err=%v", err)
if err = r.makeClient(); err != nil {
log.Error("unable to make InfluxDB client. err=%v", err)
}
}
}
}
}
func (this *WatchActord) dueJobsWithin(topic string, timeSpan int64,
now time.Time) (backlog int64, archive int64) {
jobTable := jm.JobTable(topic)
appid := manager.Default.TopicAppid(topic)
aid := jm.App_id(appid)
sql := fmt.Sprintf("SELECT count(job_id) FROM %s WHERE due_time<=?", jobTable)
rows, err := this.mc.Query(jm.AppPool, jobTable, aid, sql, now.Unix()+timeSpan)
if err != nil {
log.Error("%s: %s", this.ident(), err)
return
}
var n int
for rows.Next() {
rows.Scan(&n)
}
rows.Close()
backlog += int64(n)
archiveTable := jm.HistoryTable(topic)
sql = fmt.Sprintf("SELECT count(job_id) FROM %s WHERE due_time>=?", archiveTable)
rows, err = this.mc.Query(jm.AppPool, archiveTable, aid, sql, now.Unix()-timeSpan)
if err != nil {
log.Error("%s: %s", this.ident(), err)
return
}
for rows.Next() {
rows.Scan(&n)
}
rows.Close()
archive += int64(n)
return
}
// TODO batch DELETE/INSERT for better performance.
func (this *JobExecutor) handleDueJobs(wg *sync.WaitGroup) {
defer wg.Done()
var (
// zabbix maintains a in-memory delete queue
// delete from history_uint where itemid=? and clock<min_clock
sqlDeleteJob = fmt.Sprintf("DELETE FROM %s WHERE job_id=?", this.table)
sqlInsertArchive = fmt.Sprintf("INSERT INTO %s(job_id,payload,ctime,due_time,etime,actor_id) VALUES(?,?,?,?,?,?)",
jm.HistoryTable(this.topic))
sqlReinject = fmt.Sprintf("INSERT INTO %s(job_id, payload, ctime, due_time) VALUES(?,?,?,?)", this.table)
)
for {
select {
case <-this.stopper:
return
case item := <-this.dueJobs:
now := time.Now()
affectedRows, _, err := this.mc.Exec(jm.AppPool, this.table, this.aid, sqlDeleteJob, item.JobId)
if err != nil {
log.Error("%s: %s", this.ident, err)
continue
}
if affectedRows == 0 {
// 2 possibilities:
// - client Cancel job wins
// - this handler is too slow and the job fetched twice in tick
continue
}
log.Debug("%s land %s", this.ident, item)
_, _, err = store.DefaultPubStore.SyncPub(this.cluster, this.topic, nil, item.Payload)
if err != nil {
err = hh.Default.Append(this.cluster, this.topic, nil, item.Payload)
}
if err != nil {
// pub fails and hinted handoff also fails: reinject job back to mysql
log.Error("%s: %s", this.ident, err)
this.mc.Exec(jm.AppPool, this.table, this.aid, sqlReinject,
item.JobId, item.Payload, item.Ctime, item.DueTime)
continue
}
log.Debug("%s fired %s", this.ident, item)
this.auditor.Trace(item.String())
// mv job to archive table
_, _, err = this.mc.Exec(jm.AppPool, this.table, this.aid, sqlInsertArchive,
item.JobId, item.Payload, item.Ctime, item.DueTime, now.Unix(), this.parentId)
if err != nil {
log.Error("%s: %s", this.ident, err)
} else {
log.Debug("%s archived %s", this.ident, item)
}
}
}
}
// @rest GET /v1/partitions/:appid/:topic/:ver
func (this *manServer) partitionsHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
topic := params.ByName(UrlParamTopic)
hisAppid := params.ByName(UrlParamAppid)
appid := r.Header.Get(HttpHeaderAppid)
pubkey := r.Header.Get(HttpHeaderPubkey)
ver := params.ByName(UrlParamVersion)
realIp := getHttpRemoteIp(r)
cluster, found := manager.Default.LookupCluster(hisAppid)
if !found {
log.Error("partitions[%s] %s(%s) {app:%s topic:%s ver:%s} invalid appid",
appid, r.RemoteAddr, realIp, hisAppid, topic, ver)
writeBadRequest(w, "invalid appid")
return
}
if !manager.Default.AuthAdmin(appid, pubkey) {
log.Warn("suspicous partitions call from %s(%s) {cluster:%s app:%s key:%s topic:%s ver:%s}",
r.RemoteAddr, realIp, cluster, appid, pubkey, topic, ver)
writeAuthFailure(w, manager.ErrAuthenticationFail)
return
}
log.Info("partitions[%s] %s(%s) {cluster:%s app:%s topic:%s ver:%s}",
appid, r.RemoteAddr, realIp, cluster, hisAppid, topic, ver)
zkcluster := meta.Default.ZkCluster(cluster)
if zkcluster == nil {
log.Error("suspicous partitions call from %s(%s) {cluster:%s app:%s key:%s topic:%s ver:%s} undefined cluster",
r.RemoteAddr, realIp, cluster, appid, pubkey, topic, ver)
writeBadRequest(w, "undefined cluster")
return
}
kfk, err := sarama.NewClient(zkcluster.BrokerList(), sarama.NewConfig())
if err != nil {
log.Error("cluster[%s] %v", zkcluster.Name(), err)
writeServerError(w, err.Error())
return
}
defer kfk.Close()
partitions, err := kfk.Partitions(manager.Default.KafkaTopic(hisAppid, topic, ver))
if err != nil {
log.Error("cluster[%s] from %s(%s) {app:%s topic:%s ver:%s} %v",
zkcluster.Name(), r.RemoteAddr, realIp, hisAppid, topic, ver, err)
writeServerError(w, err.Error())
return
}
w.Write([]byte(fmt.Sprintf(`{"num": %d}`, len(partitions))))
}
func (this *WatchReplicas) report() (deadPartitions, outOfSyncPartitions int64) {
this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
brokerList := zkcluster.BrokerList()
if len(brokerList) == 0 {
log.Warn("cluster[%s] empty brokers", zkcluster.Name())
return
}
kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
if err != nil {
log.Error("cluster[%s] %v", zkcluster.Name(), err)
return
}
defer kfk.Close()
topics, err := kfk.Topics()
if err != nil {
log.Error("cluster[%s] %v", zkcluster.Name(), err)
return
}
for _, topic := range topics {
alivePartitions, err := kfk.WritablePartitions(topic)
if err != nil {
log.Error("cluster[%s] topic:%s %v", zkcluster.Name(), topic, err)
continue
}
partions, err := kfk.Partitions(topic)
if err != nil {
log.Error("cluster[%s] topic:%s %v", zkcluster.Name(), topic, err)
continue
}
// some partitions are dead
if len(alivePartitions) != len(partions) {
deadPartitions += 1
}
for _, partitionID := range alivePartitions {
replicas, err := kfk.Replicas(topic, partitionID)
if err != nil {
log.Error("cluster[%s] topic:%s partition:%d %v",
zkcluster.Name(), topic, partitionID, err)
continue
}
isr, _, _ := zkcluster.Isr(topic, partitionID)
if len(isr) != len(replicas) {
outOfSyncPartitions += 1
}
}
}
})
return
}
func fetchDashboardStats(statsUri string) (v map[string]map[string]int64) {
v = make(map[string]map[string]int64)
client := http.Client{Timeout: time.Second * 4}
resp, err := client.Get(statsUri)
if err != nil {
log.Error("%s: %v", statsUri, err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Error("%s: got status %v", statsUri, resp.Status)
return
}
reader := csv.NewReader(resp.Body)
records, err := reader.ReadAll()
if err != nil {
log.Error("%s: %v", statsUri, err)
return
}
theCols := make(map[int]string) // col:name
for i, row := range records {
if i == 0 {
// header
for j, col := range row {
theCols[j] = col
}
continue
}
if row[1] != "BACKEND" || (row[0] != "pub" && row[0] != "sub" && row[0] != "man") {
continue
}
v[row[0]] = make(map[string]int64)
for i, col := range row {
if _, present := colsMap[theCols[i]]; !present {
// ignore unwanted metrics
continue
}
if strings.HasPrefix(theCols[i], "#") || theCols[i] == "svname" {
continue
}
n, _ := strconv.ParseInt(col, 10, 64)
v[row[0]][theCols[i]] = n
}
}
return
}
// DELETE /v1/jobs/:topic/:ver?id=22323
func (this *pubServer) deleteJobHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
appid := r.Header.Get(HttpHeaderAppid)
topic := params.ByName(UrlParamTopic)
ver := params.ByName(UrlParamVersion)
realIp := getHttpRemoteIp(r)
if err := manager.Default.OwnTopic(appid, r.Header.Get(HttpHeaderPubkey), topic); err != nil {
log.Error("-job[%s] %s(%s) {topic:%s, ver:%s} %s",
appid, r.RemoteAddr, realIp, topic, ver, err)
writeAuthFailure(w, err)
return
}
_, found := manager.Default.LookupCluster(appid)
if !found {
log.Error("-job[%s] %s(%s) {topic:%s, ver:%s} cluster not found",
appid, r.RemoteAddr, realIp, topic, ver)
writeBadRequest(w, "invalid appid")
return
}
jobId := r.URL.Query().Get("id")
if len(jobId) < 18 { // jobId e,g. 341647700585877504
writeBadRequest(w, "invalid job id")
return
}
if err := job.Default.Delete(appid, manager.Default.KafkaTopic(appid, topic, ver), jobId); err != nil {
if err == job.ErrNothingDeleted {
// race failed, actor worker wins
log.Warn("-job[%s] %s(%s) {topic:%s, ver:%s jid:%s} %v",
appid, r.RemoteAddr, realIp, topic, ver, jobId, err)
w.WriteHeader(http.StatusConflict)
w.Write([]byte{})
return
}
log.Error("-job[%s] %s(%s) {topic:%s, ver:%s jid:%s} %v",
appid, r.RemoteAddr, realIp, topic, ver, jobId, err)
writeServerError(w, err.Error())
return
}
if Options.AuditPub {
this.auditor.Trace("-job[%s] %s(%s) {topic:%s ver:%s UA:%s jid:%s}",
appid, r.RemoteAddr, realIp, topic, ver, r.Header.Get("User-Agent"), jobId)
}
w.Write(ResponseOk)
}
// @rest DELETE /v1/manager/cache
func (this *manServer) refreshManagerHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
appid := r.Header.Get(HttpHeaderAppid)
pubkey := r.Header.Get(HttpHeaderPubkey)
realIp := getHttpRemoteIp(r)
if !manager.Default.AuthAdmin(appid, pubkey) {
log.Warn("suspicous refresh call from %s(%s) {app:%s key:%s}",
r.RemoteAddr, realIp, appid, pubkey)
writeAuthFailure(w, manager.ErrAuthenticationFail)
return
}
if !this.throttleAddTopic.Pour(realIp, 1) {
writeQuotaExceeded(w)
return
}
kateways, err := this.gw.zkzone.KatewayInfos()
if err != nil {
log.Error("refresh from %s(%s) %v", r.RemoteAddr, getHttpRemoteIp(r), err)
writeServerError(w, err.Error())
return
}
// refresh locally
manager.Default.ForceRefresh()
// refresh zone wide
allOk := true
for _, kw := range kateways {
if kw.Id != this.gw.id {
// notify other kateways to refresh: avoid dead loop in the network
if err := this.gw.callKateway(kw, "PUT", "v1/options/refreshdb/true"); err != nil {
// don't retry, just log
log.Error("refresh from %s(%s) %s@%s: %v", r.RemoteAddr, realIp, kw.Id, kw.Host, err)
allOk = false
}
}
}
log.Info("refresh from %s(%s) all ok: %v", r.RemoteAddr, realIp, allOk)
if !allOk {
writeServerError(w, "cache partially refreshed")
return
}
w.Write(ResponseOk)
}
func (this *WatchInfluxServer) Run() {
defer this.Wg.Done()
if this.addr == "" {
log.Warn("empty influxdb server addr, quit...")
return
}
ticker := time.NewTicker(this.Tick)
defer ticker.Stop()
var (
err error
influxdbServerAlive = metrics.NewRegisteredGauge("influxdb.alive", nil)
)
for {
select {
case <-this.Stop:
log.Info("influx.server stopped")
return
case <-ticker.C:
if this.cli == nil {
this.cli, err = client.NewHTTPClient(client.HTTPConfig{
Addr: this.addr,
Username: "",
Password: "",
})
if err != nil {
log.Error("influxdb.server: %v", err)
influxdbServerAlive.Update(0)
continue
}
if this.cli == nil {
log.Error("influxdb.server connected got nil cli")
influxdbServerAlive.Update(0)
continue
}
}
_, _, err = this.cli.Ping(time.Second * 4)
if err != nil {
log.Error("influxdb.server: %v", err)
influxdbServerAlive.Update(0)
} else {
influxdbServerAlive.Update(1)
}
}
}
}
func (this *WatchRedisInfo) updateRedisInfo(wg *sync.WaitGroup, host string, port int, tag string) {
defer wg.Done()
spec := redis.DefaultSpec().Host(host).Port(port)
client, err := redis.NewSynchClientWithSpec(spec)
if err != nil {
log.Error("redis[%s:%d]: %v", host, port, err)
atomic.AddInt64(&this.deadN, 1)
return
}
defer client.Quit()
infoMap, err := client.Info()
if err != nil {
log.Error("redis[%s:%d] info: %v", host, port, err)
atomic.AddInt64(&this.deadN, 1)
return
}
var keysN int64
// db0:keys=15500,expires=15500,avg_ttl=27438570
for key, value := range infoMap {
if strings.HasPrefix(key, "db") && strings.Contains(value, "keys=") {
keysN += extractKeysCount(value)
}
}
conns, _ := strconv.ParseInt(infoMap["connected_clients"], 10, 64)
blocked, _ := strconv.ParseInt(infoMap["blocked_clients"], 10, 64)
mem, _ := strconv.ParseInt(infoMap["used_memory"], 10, 64)
ops, _ := strconv.ParseInt(infoMap["instantaneous_ops_per_sec"], 10, 64)
rejected, _ := strconv.ParseInt(infoMap["rejected_connections"], 10, 64)
syncPartial, _ := strconv.ParseInt(infoMap["sync_partial_err"], 10, 64)
rxKbps, _ := strconv.ParseFloat(infoMap["instantaneous_input_kbps"], 64)
txKbps, _ := strconv.ParseFloat(infoMap["instantaneous_output_kbps"], 64)
expiredKeys, _ := strconv.ParseInt(infoMap["expired_keys"], 10, 64)
atomic.AddInt64(&this.syncPartialN, syncPartial)
this.mu.Lock()
this.keys[tag].Update(keysN)
this.conns[tag].Update(conns)
this.blocked[tag].Update(blocked)
this.usedMem[tag].Update(mem)
this.ops[tag].Update(ops)
this.rejected[tag].Update(rejected)
this.rxKbps[tag].Update(int64(rxKbps))
this.txKbps[tag].Update(int64(txKbps))
this.expiredKeys[tag].Update(expiredKeys)
this.mu.Unlock()
}
func (this *WatchSlowlog) Run() {
defer this.Wg.Done()
ticker := time.NewTicker(this.Tick)
defer ticker.Stop()
this.slows = make(map[string]metrics.Gauge, 10)
for {
select {
case <-this.Stop:
log.Info("redis.slowlog stopped")
return
case <-ticker.C:
var wg sync.WaitGroup
for _, hostPort := range this.Zkzone.AllRedis() {
host, port, err := net.SplitHostPort(hostPort)
if err != nil {
log.Error("invalid redis instance: %s", hostPort)
continue
}
nport, err := strconv.Atoi(port)
if err != nil || nport < 0 {
log.Error("invalid redis instance: %s", hostPort)
continue
}
var ip string
ips, err := net.LookupIP(host) // host in ip form is also ok e,g. 10.1.1.1
if err != nil {
log.Error("redis host[%s] ip: %v", host, err)
} else if len(ips) > 0 {
ip = ips[0].String()
}
tag := telemetry.Tag(strings.Replace(host, ".", "_", -1), port, ip)
wg.Add(1)
go this.updateRedisSlowlog(&wg, host, nport, tag)
}
wg.Wait()
}
}
}
func (this *WatchExec) Run() {
defer this.Wg.Done()
if this.confDir == "" {
log.Warn("empty confd, external.exec disabled")
return
}
ticker := time.NewTicker(time.Minute)
defer ticker.Stop()
if err := this.watchConfigDir(); err != nil {
log.Error("%v", err)
return
}
for {
select {
case <-this.Stop:
log.Info("external.exec stopped")
return
case <-ticker.C:
}
}
}
// append raw data to an existing item
func (this *FunServantImpl) CbAppend(ctx *rpc.Context, bucket string,
key string, val []byte) (ex error) {
const IDENT = "cb.append"
if this.cb == nil {
ex = ErrServantNotStarted
return
}
profiler, err := this.getSession(ctx).startProfiler()
if err != nil {
ex = err
return
}
svtStats.inc(IDENT)
b, err := this.cb.GetBucket(bucket)
if err != nil {
ex = err
return
}
ex = b.Append(key, val)
if ex != nil {
log.Error("Q=%s %s: %s %s", IDENT, ctx.String(), key, ex)
}
profiler.do(IDENT, ctx,
"{b^%s k^%s v^%s}",
bucket, key, string(val))
return
}
func (this *FunServantImpl) CbAdd(ctx *rpc.Context, bucket string,
key string, val []byte, expire int32) (r bool, ex error) {
const IDENT = "cb.add"
if this.cb == nil {
ex = ErrServantNotStarted
return
}
profiler, err := this.getSession(ctx).startProfiler()
if err != nil {
ex = err
return
}
svtStats.inc(IDENT)
b, err := this.cb.GetBucket(bucket)
if err != nil {
ex = err
return
}
r, ex = b.AddRaw(key, int(expire), val)
if ex != nil {
log.Error("Q=%s %s: %s %s", IDENT, ctx.String(), key, ex)
}
profiler.do(IDENT, ctx,
"{b^%s k^%s v^%s exp^%d} {r^%v}",
bucket, key, string(val), expire, r)
return
}
func (this *Peer) discoverPeers() {
defer func() {
this.c.Close() // leave the multicast group
}()
var msg peerMessage
reader := bufio.NewReader(this.c)
for {
// net.ListenMulticastUDP sets IP_MULTICAST_LOOP=0 as
// default, so you never receive your own sent data
// if you run both sender and receiver on (logically) same IP host
line, _, err := reader.ReadLine()
if err != nil {
log.Error(err)
continue
}
if err := msg.unmarshal(line); err != nil {
// Not our protocol, it may be SSDP or else
continue
}
log.Debug("received peer: %+v", msg)
neighborIp, present := msg["ip"]
if !present {
log.Info("Peer msg has no 'ip'")
continue
}
this.refreshNeighbor(neighborIp.(string))
}
}
func (this *mysqlStore) fetchTopicRecords(db *sql.DB) error {
rows, err := db.Query("SELECT AppId,TopicName,Status FROM topics")
if err != nil {
return err
}
defer rows.Close()
m := make(map[string]map[string]bool)
var app appTopicRecord
for rows.Next() {
err = rows.Scan(&app.AppId, &app.TopicName, &app.Status)
if err != nil {
log.Error("mysql manager store: %v", err)
continue
}
if _, present := m[app.AppId]; !present {
m[app.AppId] = make(map[string]bool)
}
m[app.AppId][app.TopicName] = app.Status == "1"
}
this.appTopicsMap = m
return nil
}