func analyzeGardenAUFSStressTests() {
runs := []string{"diego-2.cell-z1.0"} //, "diego-1.cell-z1.0"}
for _, run := range runs {
data, err := ioutil.ReadFile(config.DataDir("garden-aufs-test", run+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
// significantEvents.LogWithThreshold(0.2)
allow := map[string]bool{
"garden-linux.loop-mounter.unmount.failed-to-unmount": true,
"garden-linux.garden-server.create.creating": true,
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.depot-client.run-container.creating-container-in-garden": true,
"rep.depot-client.delete-container.destroy.started": true,
"rep.depot-client.run-container.run.action.download-step.fetch-starting": true,
"rep.depot-client.run-container.run.monitor-run.run-step.running": true,
"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
"rep.depot-client.run-container.run.setup.download-step.fetch-starting": true,
"rep.auction-delegate.auction-work.lrp-allocate-instances.requesting-container-allocation": true,
"rep.event-consumer.operation-stream.executing-container-operation.task-processor.fetching-container-result": true,
"rep.depot-client.run-container.run.action.run-step.running": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
filteredSignificantEvents.LogWithThreshold(0.2)
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
limit := viz.NewHorizontalLine(256)
limit.LineStyle = viz.LineStyle(viz.Red, 1)
options := analyzers.SignificantEventsOptions{
LineOverlays: gardenStressTestContainerCountOverlays(entries),
VerticalMarkers: gardenStressTestFailedContainerCreates(entries),
OverlayPlots: []plot.Plotter{limit},
WidthStretch: 2,
MaxY: 400,
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("garden-aufs-test", run+".png"),
options,
)
}
}
func analyzePWSSlowEvacuation() {
runs := []string{"cell_z1.16", "cell_z1.17", "cell_z2.23", "cell_z2.24", "cell_z2.25"}
for _, run := range runs {
data, err := ioutil.ReadFile(config.DataDir("pws-slow-evacuation", run+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
// significantEvents.LogWithThreshold(0.2)
allow := map[string]bool{
"garden-linux.loop-mounter.unmount.failed-to-unmount": true,
"garden-linux.garden-server.create.creating": true,
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.depot-client.run-container.creating-container-in-garden": true,
"rep.depot-client.delete-container.destroy.started": true,
"rep.depot-client.run-container.run.action.download-step.fetch-starting": true,
"rep.depot-client.run-container.run.monitor-run.run-step.running": true,
"rep.depot-client.run-container.run.setup.download-step.fetch-starting": true,
"rep.auction-delegate.auction-work.lrp-allocate-instances.requesting-container-allocation": true,
"rep.event-consumer.operation-stream.executing-container-operation.task-processor.fetching-container-result": true,
"rep.depot-client.run-container.run.action.run-step.running": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
filteredSignificantEvents.LogWithThreshold(0.2)
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
options := analyzers.SignificantEventsOptions{
LineOverlays: gardenStressTestContainerCountOverlays(entries),
VerticalMarkers: pwsSlowEvacuationFailedProcessOverlays(entries),
WidthStretch: 2,
MaxY: 400,
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("pws-slow-evacuation", run+".png"),
options,
)
}
}
func analyzeSlowPWSTasks() {
for _, run := range slowPWSTaskRuns {
say.Println(0, say.Green(run.Name))
data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
allow := map[string]bool{
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.depot-client.run-container.creating-container-in-garden": true,
"rep.depot-client.delete-container.destroy.started": true,
"rep.depot-client.run-container.run.action.download-step.fetch-starting": true,
"rep.depot-client.run-container.run.monitor-run.run-step.running": true,
"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
"rep.depot-client.run-container.run.setup.download-step.fetch-starting": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
options := analyzers.SignificantEventsOptions{
LineOverlays: containerCountOverlays(entries),
}
if !run.CliffTimestamp.IsZero() {
options.MaxT = run.EndTimestamp.Add(time.Minute * 30)
options.VerticalMarkers = []analyzers.VerticalMarker{
{T: run.CliffTimestamp, LineStyle: viz.LineStyle(viz.Red, 1, viz.Dash)},
{T: run.EndTimestamp, LineStyle: viz.LineStyle(viz.Black, 1, viz.Dash)},
}
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("pws-slow-tasks", run.Name+".png"),
options,
)
}
}
func analyzeCPUWeightStresstest() {
runs := []string{
"unmodified-run",
"aufs-run",
"2-conc-run",
}
for _, run := range runs {
say.Println(0, say.Green(run))
data, err := ioutil.ReadFile(config.DataDir("cpu-wait-stress-test", run+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
allow := map[string]bool{
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.depot-client.run-container.creating-container-in-garden": true,
"rep.depot-client.delete-container.destroy.started": true,
"rep.depot-client.run-container.run.action.download-step.fetch-starting": true,
"rep.depot-client.run-container.run.monitor-run.run-step.running": true,
"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
"rep.depot-client.run-container.run.setup.download-step.fetch-starting": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
options := analyzers.SignificantEventsOptions{
LineOverlays: cpuWeightStressTestContainerCountOverlays(entries),
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("cpu-wait-stress-test", run+".png"),
options,
)
}
}
func analyzeHealthCheckTimeouts() {
data, err := ioutil.ReadFile(config.DataDir("health-check-timeouts", "health-check-timeouts.log"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
significantEvents.LogWithThreshold(0.2)
allow := map[string]bool{
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-run.node-run.monitor-run.run-step.running": true,
"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-create.starting": true,
"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-completed-container.deleting-container": true,
"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-run.node-run.action.run-step.running": true,
"rep.running-bulker.sync.starting": true,
"garden-linux.garden-server.run.spawned": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
options := analyzers.SignificantEventsOptions{
LineOverlays: gardenStressTestContainerCountOverlays(entries),
VerticalMarkers: healthCheckTimeoutsFailedHealthMonitor(entries),
WidthStretch: 2,
MaxY: 400,
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("health-check-timeouts", "health-check-timeouts.png"),
options,
)
}
func analyzeGardenStressTests() {
data, err := ioutil.ReadFile(config.DataDir("garden-stress-test", "all.unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
allow := map[string]bool{
"garden-linux.garden-server.create.creating": true,
"rep.depot-client.delete-container.destroy.started": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
filteredSignificantEvents.LogWithThreshold(0.2)
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
limit := viz.NewHorizontalLine(256)
limit.LineStyle = viz.LineStyle(viz.Red, 1)
options := analyzers.SignificantEventsOptions{
LineOverlays: gardenStressTestContainerCountOverlays(entries),
VerticalMarkers: gardenStressTestFailedContainerCreates(entries),
OverlayPlots: []plot.Plotter{limit},
WidthStretch: 3,
MaxY: 400,
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("garden-stress-test", "out.png"),
options,
)
}