Beispiel #1
0
func analyzeGardenAUFSStressTests() {
	runs := []string{"diego-2.cell-z1.0"} //, "diego-1.cell-z1.0"}
	for _, run := range runs {
		data, err := ioutil.ReadFile(config.DataDir("garden-aufs-test", run+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)
		// significantEvents.LogWithThreshold(0.2)

		allow := map[string]bool{
			"garden-linux.loop-mounter.unmount.failed-to-unmount":                                                        true,
			"garden-linux.garden-server.create.creating":                                                                 true,
			"rep.auction-fetch-state.handling":                                                                           true,
			"rep.container-metrics-reporter.tick.started":                                                                true,
			"rep.depot-client.run-container.creating-container-in-garden":                                                true,
			"rep.depot-client.delete-container.destroy.started":                                                          true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":                                     true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":                                            true,
			"rep.depot-client.run-container.run.run-step-process.step-finished-with-error":                               true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":                                      true,
			"rep.auction-delegate.auction-work.lrp-allocate-instances.requesting-container-allocation":                   true,
			"rep.event-consumer.operation-stream.executing-container-operation.task-processor.fetching-container-result": true,
			"rep.depot-client.run-container.run.action.run-step.running":                                                 true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		filteredSignificantEvents.LogWithThreshold(0.2)
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		limit := viz.NewHorizontalLine(256)
		limit.LineStyle = viz.LineStyle(viz.Red, 1)

		options := analyzers.SignificantEventsOptions{
			LineOverlays:    gardenStressTestContainerCountOverlays(entries),
			VerticalMarkers: gardenStressTestFailedContainerCreates(entries),
			OverlayPlots:    []plot.Plotter{limit},
			WidthStretch:    2,
			MaxY:            400,
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("garden-aufs-test", run+".png"),
			options,
		)
	}
}
Beispiel #2
0
func analyzePWSSlowEvacuation() {
	runs := []string{"cell_z1.16", "cell_z1.17", "cell_z2.23", "cell_z2.24", "cell_z2.25"}
	for _, run := range runs {
		data, err := ioutil.ReadFile(config.DataDir("pws-slow-evacuation", run+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)
		// significantEvents.LogWithThreshold(0.2)

		allow := map[string]bool{
			"garden-linux.loop-mounter.unmount.failed-to-unmount":                                                        true,
			"garden-linux.garden-server.create.creating":                                                                 true,
			"rep.auction-fetch-state.handling":                                                                           true,
			"rep.container-metrics-reporter.tick.started":                                                                true,
			"rep.depot-client.run-container.creating-container-in-garden":                                                true,
			"rep.depot-client.delete-container.destroy.started":                                                          true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":                                     true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":                                            true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":                                      true,
			"rep.auction-delegate.auction-work.lrp-allocate-instances.requesting-container-allocation":                   true,
			"rep.event-consumer.operation-stream.executing-container-operation.task-processor.fetching-container-result": true,
			"rep.depot-client.run-container.run.action.run-step.running":                                                 true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		filteredSignificantEvents.LogWithThreshold(0.2)
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		options := analyzers.SignificantEventsOptions{
			LineOverlays:    gardenStressTestContainerCountOverlays(entries),
			VerticalMarkers: pwsSlowEvacuationFailedProcessOverlays(entries),
			WidthStretch:    2,
			MaxY:            400,
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("pws-slow-evacuation", run+".png"),
			options,
		)
	}
}
Beispiel #3
0
func analyzeSlowPWSTasks() {
	for _, run := range slowPWSTaskRuns {
		say.Println(0, say.Green(run.Name))
		data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)

		allow := map[string]bool{
			"rep.auction-fetch-state.handling":                                             true,
			"rep.container-metrics-reporter.tick.started":                                  true,
			"rep.depot-client.run-container.creating-container-in-garden":                  true,
			"rep.depot-client.delete-container.destroy.started":                            true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":       true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":              true,
			"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":        true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		options := analyzers.SignificantEventsOptions{
			LineOverlays: containerCountOverlays(entries),
		}
		if !run.CliffTimestamp.IsZero() {
			options.MaxT = run.EndTimestamp.Add(time.Minute * 30)
			options.VerticalMarkers = []analyzers.VerticalMarker{
				{T: run.CliffTimestamp, LineStyle: viz.LineStyle(viz.Red, 1, viz.Dash)},
				{T: run.EndTimestamp, LineStyle: viz.LineStyle(viz.Black, 1, viz.Dash)},
			}
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("pws-slow-tasks", run.Name+".png"),
			options,
		)

	}
}
Beispiel #4
0
func analyzeCPUWeightStresstest() {
	runs := []string{
		"unmodified-run",
		"aufs-run",
		"2-conc-run",
	}
	for _, run := range runs {
		say.Println(0, say.Green(run))
		data, err := ioutil.ReadFile(config.DataDir("cpu-wait-stress-test", run+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)

		allow := map[string]bool{
			"rep.auction-fetch-state.handling":                                             true,
			"rep.container-metrics-reporter.tick.started":                                  true,
			"rep.depot-client.run-container.creating-container-in-garden":                  true,
			"rep.depot-client.delete-container.destroy.started":                            true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":       true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":              true,
			"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":        true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		options := analyzers.SignificantEventsOptions{
			LineOverlays: cpuWeightStressTestContainerCountOverlays(entries),
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("cpu-wait-stress-test", run+".png"),
			options,
		)

	}
}
Beispiel #5
0
func analyzeHealthCheckTimeouts() {
	data, err := ioutil.ReadFile(config.DataDir("health-check-timeouts", "health-check-timeouts.log"))
	say.ExitIfError("couldn't read log file", err)

	entries := util.ChugLagerEntries(data)

	significantEvents := analyzers.ExtractSignificantEvents(entries)
	significantEvents.LogWithThreshold(0.2)

	allow := map[string]bool{
		"rep.auction-fetch-state.handling":            true,
		"rep.container-metrics-reporter.tick.started": true,
		"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-run.node-run.monitor-run.run-step.running": true,
		"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-create.starting":                           true,
		"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-completed-container.deleting-container":                                                    true,
		"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-run.node-run.action.run-step.running":      true,
		"rep.running-bulker.sync.starting":       true,
		"garden-linux.garden-server.run.spawned": true,
	}

	filteredSignificantEvents := analyzers.SignificantEvents{}

	for name, events := range significantEvents {
		if allow[name] {
			filteredSignificantEvents[name] = events
		}
	}
	filteredSignificantEvents.LogWithThreshold(0.2)

	options := analyzers.SignificantEventsOptions{
		LineOverlays:    gardenStressTestContainerCountOverlays(entries),
		VerticalMarkers: healthCheckTimeoutsFailedHealthMonitor(entries),
		WidthStretch:    2,
		MaxY:            400,
	}

	analyzers.VisualizeSignificantEvents(
		filteredSignificantEvents,
		config.DataDir("health-check-timeouts", "health-check-timeouts.png"),
		options,
	)
}
Beispiel #6
0
func analyzeGardenStressTests() {
	data, err := ioutil.ReadFile(config.DataDir("garden-stress-test", "all.unified"))
	say.ExitIfError("couldn't read log file", err)

	entries := util.ChugLagerEntries(data)

	significantEvents := analyzers.ExtractSignificantEvents(entries)

	allow := map[string]bool{
		"garden-linux.garden-server.create.creating":        true,
		"rep.depot-client.delete-container.destroy.started": true,
	}

	filteredSignificantEvents := analyzers.SignificantEvents{}
	filteredSignificantEvents.LogWithThreshold(0.2)
	for name, events := range significantEvents {
		if allow[name] {
			filteredSignificantEvents[name] = events
		}
	}
	filteredSignificantEvents.LogWithThreshold(0.2)

	limit := viz.NewHorizontalLine(256)
	limit.LineStyle = viz.LineStyle(viz.Red, 1)

	options := analyzers.SignificantEventsOptions{
		LineOverlays:    gardenStressTestContainerCountOverlays(entries),
		VerticalMarkers: gardenStressTestFailedContainerCreates(entries),
		OverlayPlots:    []plot.Plotter{limit},
		WidthStretch:    3,
		MaxY:            400,
	}

	analyzers.VisualizeSignificantEvents(
		filteredSignificantEvents,
		config.DataDir("garden-stress-test", "out.png"),
		options,
	)
}