func EmitRunOnceStates(datadogClient *datadog.Client, store *etcdstoreadapter.ETCDStoreAdapter, etcdMachines []string) {
for {
now := time.Now().Unix()
all, err := store.ListRecursively("/v1/run_once")
if err != nil {
log.Println("failed to get all RunOnces:", err)
time.Sleep(1 * time.Second)
continue
}
metrics := []datadog.Metric{}
for i, etcdMachine := range etcdMachines {
stats := map[string]int{}
resp, err := http.Get(urljoiner.Join(etcdMachine, "/v2/stats/store"))
if err != nil {
log.Println("failed to fetch stats:", err)
continue
}
data, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
json.Unmarshal(data, &stats)
metrics = append(metrics, datadog.Metric{
Metric: fmt.Sprintf("etcd_watchers_%d", i),
Points: []datadog.DataPoint{
datadog.DataPoint(
[2]float64{
float64(now),
float64(stats["watchers"]),
},
),
},
})
}
for _, state := range []string{"pending", "claimed", "running", "completed", "resolving"} {
runOnces, found := all.Lookup(state)
if !found {
log.Println("failed to find RunOnces in", state, "state")
time.Sleep(1 * time.Second)
continue
}
metrics = append(metrics, datadog.Metric{
Metric: "diego_runonce_" + state,
Points: []datadog.DataPoint{
datadog.DataPoint(
[2]float64{
float64(now),
float64(len(runOnces.ChildNodes)),
},
),
},
})
}
executors, err := store.ListRecursively("/v1/executor")
if err != nil {
log.Println("failed to get all Executors:", err)
time.Sleep(1 * time.Second)
continue
}
metrics = append(metrics, datadog.Metric{
Metric: "executors_maintaining_presence",
Points: []datadog.DataPoint{
datadog.DataPoint(
[2]float64{
float64(now),
float64(len(executors.ChildNodes)),
},
),
},
})
err = datadogClient.PostMetrics(metrics)
if err != nil {
log.Println("failed to post metrics:", err)
}
time.Sleep(1 * time.Second)
}
}
