func EmitRunOnceStates(datadogClient *datadog.Client, store *etcdstoreadapter.ETCDStoreAdapter, etcdMachines []string) {
for {
now := time.Now().Unix()
all, err := store.ListRecursively("/v1/run_once")
if err != nil {
log.Println("failed to get all RunOnces:", err)
time.Sleep(1 * time.Second)
continue
}
metrics := []datadog.Metric{}
for i, etcdMachine := range etcdMachines {
stats := map[string]int{}
resp, err := http.Get(urljoiner.Join(etcdMachine, "/v2/stats/store"))
if err != nil {
log.Println("failed to fetch stats:", err)
continue
}
data, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
json.Unmarshal(data, &stats)
metrics = append(metrics, datadog.Metric{
Metric: fmt.Sprintf("etcd_watchers_%d", i),
Points: []datadog.DataPoint{
datadog.DataPoint(
[2]float64{
float64(now),
float64(stats["watchers"]),
},
),
},
})
}
for _, state := range []string{"pending", "claimed", "running", "completed", "resolving"} {
runOnces, found := all.Lookup(state)
if !found {
log.Println("failed to find RunOnces in", state, "state")
time.Sleep(1 * time.Second)
continue
}
metrics = append(metrics, datadog.Metric{
Metric: "diego_runonce_" + state,
Points: []datadog.DataPoint{
datadog.DataPoint(
[2]float64{
float64(now),
float64(len(runOnces.ChildNodes)),
},
),
},
})
}
executors, err := store.ListRecursively("/v1/executor")
if err != nil {
log.Println("failed to get all Executors:", err)
time.Sleep(1 * time.Second)
continue
}
metrics = append(metrics, datadog.Metric{
Metric: "executors_maintaining_presence",
Points: []datadog.DataPoint{
datadog.DataPoint(
[2]float64{
float64(now),
float64(len(executors.ChildNodes)),
},
),
},
})
err = datadogClient.PostMetrics(metrics)
if err != nil {
log.Println("failed to post metrics:", err)
}
time.Sleep(1 * time.Second)
}
}
func RunonceStampede(bbs *bbs.BBS, datadogClient *datadog.Client, runOnce *models.RunOnce, runOnceCount int) {
completed, stop, errs := bbs.WatchForCompletedRunOnce()
startAll := time.Now()
if datadogClient != nil {
event, err := datadogClient.PostEvent(&datadog.Event{
Title: "diego_runonce_stampede_start",
Text: "started the stampede",
Tags: []string{fmt.Sprintf("count:%d", runOnceCount)},
})
log.Println("posted start event:", event, err)
defer func() {
event, err := datadogClient.PostEvent(&datadog.Event{
Title: "diego_runonce_stampede_stop",
Text: "stopped the stampede",
Tags: []string{
fmt.Sprintf("count:%d", runOnceCount),
fmt.Sprintf("duration:%s", time.Since(startAll)),
},
})
log.Println("posted stop event:", event, err)
}()
}
startTimes := make(chan runOnceTime, runOnceCount)
go func() {
for i := 0; i < runOnceCount; i++ {
go createRunOnce(runOnce, startTimes, bbs)
}
}()
seenRunOnces := 0
runOnceStartTimes := make(map[string]time.Time)
waitGroup := &sync.WaitGroup{}
timer := time.After(100 * time.Minute)
OUTER:
for {
if seenRunOnces == runOnceCount {
timer = time.After(30 * time.Second)
}
select {
case startTime := <-startTimes:
runOnceStartTimes[startTime.guid] = startTime.startTime
case completedRunOnce := <-completed:
startedAt, found := runOnceStartTimes[completedRunOnce.Guid]
if !found {
continue
}
log.Println("done:", seenRunOnces, RunOnceResult{
Guid: completedRunOnce.Guid,
Duration: time.Since(startedAt),
Failed: completedRunOnce.Failed,
})
seenRunOnces++
waitGroup.Add(1)
go func() {
log.Println("deleting", completedRunOnce.Guid)
err := bbs.ResolveRunOnce(completedRunOnce)
if err != nil {
log.Println("failed to resolve run once:", completedRunOnce.Guid, err)
} else {
log.Println("deleted:", completedRunOnce.Guid)
}
waitGroup.Done()
}()
case err := <-errs:
log.Println("watch error:", err)
case <-timer:
break OUTER
}
}
waitGroup.Wait()
close(stop)
}