Beispiel #1
0
func newControllerFromEnv() controller.Controller {
	const atlasBaseURL = "https://atlas.hashicorp.com"
	const boxName = "cloudfoundry/bosh-lite"

	awsRegion := loadOrFail("AWS_DEFAULT_REGION")
	templateBody, err := json.Marshal(templates.DefaultTemplate)
	say.ExitIfError("internal error: unable to marshal CloudFormation template", err)

	webClient := &client.WebClient{}
	jsonClient := client.JSONClient{BaseURL: atlasBaseURL}
	atlasClient := &client.AtlasClient{&jsonClient}
	awsClient, err := aws.New(aws.Config{
		AccessKey:  loadOrFail("AWS_ACCESS_KEY_ID"),
		SecretKey:  loadOrFail("AWS_SECRET_ACCESS_KEY"),
		RegionName: awsRegion,
	})
	say.ExitIfError("internal error: unable to create AWS client", err)
	parallelRunner := &shell.ParallelRunner{Runner: &shell.Runner{}}

	controller := controller.Controller{
		AtlasClient:    atlasClient,
		AWSClient:      awsClient,
		Log:            &CliLogger{},
		WebClient:      webClient,
		ParallelRunner: parallelRunner,

		VagrantBoxName: boxName,
		Region:         awsRegion,
		Template:       string(templateBody),
		SSHPort:        22,
		SSHUser:        "******",
	}

	return controller
}
Beispiel #2
0
func loadOrFail(varName string) string {
	val := os.Getenv(varName)
	if val == "" {
		say.ExitIfError("Missing required environment variable", fmt.Errorf("'%s'", varName))
	}
	return val
}
Beispiel #3
0
func validateRequiredArgument(variableName string, value interface{}) {
	notSet := (value == reflect.Zero(reflect.TypeOf(value)).Interface())

	if notSet {
		say.ExitIfError("Missing required argument", errors.New("'"+variableName+"'"))
	}
}
Beispiel #4
0
func analyzeGardenAUFSStressTests() {
	runs := []string{"diego-2.cell-z1.0"} //, "diego-1.cell-z1.0"}
	for _, run := range runs {
		data, err := ioutil.ReadFile(config.DataDir("garden-aufs-test", run+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)
		// significantEvents.LogWithThreshold(0.2)

		allow := map[string]bool{
			"garden-linux.loop-mounter.unmount.failed-to-unmount":                                                        true,
			"garden-linux.garden-server.create.creating":                                                                 true,
			"rep.auction-fetch-state.handling":                                                                           true,
			"rep.container-metrics-reporter.tick.started":                                                                true,
			"rep.depot-client.run-container.creating-container-in-garden":                                                true,
			"rep.depot-client.delete-container.destroy.started":                                                          true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":                                     true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":                                            true,
			"rep.depot-client.run-container.run.run-step-process.step-finished-with-error":                               true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":                                      true,
			"rep.auction-delegate.auction-work.lrp-allocate-instances.requesting-container-allocation":                   true,
			"rep.event-consumer.operation-stream.executing-container-operation.task-processor.fetching-container-result": true,
			"rep.depot-client.run-container.run.action.run-step.running":                                                 true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		filteredSignificantEvents.LogWithThreshold(0.2)
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		limit := viz.NewHorizontalLine(256)
		limit.LineStyle = viz.LineStyle(viz.Red, 1)

		options := analyzers.SignificantEventsOptions{
			LineOverlays:    gardenStressTestContainerCountOverlays(entries),
			VerticalMarkers: gardenStressTestFailedContainerCreates(entries),
			OverlayPlots:    []plot.Plotter{limit},
			WidthStretch:    2,
			MaxY:            400,
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("garden-aufs-test", run+".png"),
			options,
		)
	}
}
Beispiel #5
0
func analyzePWSSlowEvacuation() {
	runs := []string{"cell_z1.16", "cell_z1.17", "cell_z2.23", "cell_z2.24", "cell_z2.25"}
	for _, run := range runs {
		data, err := ioutil.ReadFile(config.DataDir("pws-slow-evacuation", run+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)
		// significantEvents.LogWithThreshold(0.2)

		allow := map[string]bool{
			"garden-linux.loop-mounter.unmount.failed-to-unmount":                                                        true,
			"garden-linux.garden-server.create.creating":                                                                 true,
			"rep.auction-fetch-state.handling":                                                                           true,
			"rep.container-metrics-reporter.tick.started":                                                                true,
			"rep.depot-client.run-container.creating-container-in-garden":                                                true,
			"rep.depot-client.delete-container.destroy.started":                                                          true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":                                     true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":                                            true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":                                      true,
			"rep.auction-delegate.auction-work.lrp-allocate-instances.requesting-container-allocation":                   true,
			"rep.event-consumer.operation-stream.executing-container-operation.task-processor.fetching-container-result": true,
			"rep.depot-client.run-container.run.action.run-step.running":                                                 true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		filteredSignificantEvents.LogWithThreshold(0.2)
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		options := analyzers.SignificantEventsOptions{
			LineOverlays:    gardenStressTestContainerCountOverlays(entries),
			VerticalMarkers: pwsSlowEvacuationFailedProcessOverlays(entries),
			WidthStretch:    2,
			MaxY:            400,
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("pws-slow-evacuation", run+".png"),
			options,
		)
	}
}
Beispiel #6
0
func analyzeSlowPWSTasks() {
	for _, run := range slowPWSTaskRuns {
		say.Println(0, say.Green(run.Name))
		data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)

		allow := map[string]bool{
			"rep.auction-fetch-state.handling":                                             true,
			"rep.container-metrics-reporter.tick.started":                                  true,
			"rep.depot-client.run-container.creating-container-in-garden":                  true,
			"rep.depot-client.delete-container.destroy.started":                            true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":       true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":              true,
			"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":        true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		options := analyzers.SignificantEventsOptions{
			LineOverlays: containerCountOverlays(entries),
		}
		if !run.CliffTimestamp.IsZero() {
			options.MaxT = run.EndTimestamp.Add(time.Minute * 30)
			options.VerticalMarkers = []analyzers.VerticalMarker{
				{T: run.CliffTimestamp, LineStyle: viz.LineStyle(viz.Red, 1, viz.Dash)},
				{T: run.EndTimestamp, LineStyle: viz.LineStyle(viz.Black, 1, viz.Dash)},
			}
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("pws-slow-tasks", run.Name+".png"),
			options,
		)

	}
}
Beispiel #7
0
func analyzeCPUWeightStresstest() {
	runs := []string{
		"unmodified-run",
		"aufs-run",
		"2-conc-run",
	}
	for _, run := range runs {
		say.Println(0, say.Green(run))
		data, err := ioutil.ReadFile(config.DataDir("cpu-wait-stress-test", run+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)

		allow := map[string]bool{
			"rep.auction-fetch-state.handling":                                             true,
			"rep.container-metrics-reporter.tick.started":                                  true,
			"rep.depot-client.run-container.creating-container-in-garden":                  true,
			"rep.depot-client.delete-container.destroy.started":                            true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":       true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":              true,
			"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":        true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		options := analyzers.SignificantEventsOptions{
			LineOverlays: cpuWeightStressTestContainerCountOverlays(entries),
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("cpu-wait-stress-test", run+".png"),
			options,
		)

	}
}
Beispiel #8
0
func NewListCommand() say.Command {
	var format string
	flags := flag.NewFlagSet("list", flag.ContinueOnError)

	flags.StringVar(&format, "format", "json", "output format: json or plain")

	return say.Command{
		Name:        "list",
		Description: "List all classrooms",
		FlagSet:     flags,
		Run: func(args []string) {

			c := newControllerFromEnv()
			output, err := c.ListClassrooms(format)
			say.ExitIfError("Failed listing classrooms", err)
			fmt.Println(output)
		},
	}
}
Beispiel #9
0
func NewDestroyCommand() say.Command {
	var name string

	flags := flag.NewFlagSet("destroy", flag.ContinueOnError)

	flags.StringVar(&name, "name", "", "classroom name")

	return say.Command{
		Name:        "destroy",
		Description: "Destroy an existing classroom",
		FlagSet:     flags,
		Run: func(args []string) {
			validateRequiredArgument("name", name)
			c := newControllerFromEnv()
			err := c.DestroyClassroom(name)
			say.ExitIfError("Failed while destroying classroom", err)
		},
	}
}
Beispiel #10
0
func analyzeHealthCheckTimeouts() {
	data, err := ioutil.ReadFile(config.DataDir("health-check-timeouts", "health-check-timeouts.log"))
	say.ExitIfError("couldn't read log file", err)

	entries := util.ChugLagerEntries(data)

	significantEvents := analyzers.ExtractSignificantEvents(entries)
	significantEvents.LogWithThreshold(0.2)

	allow := map[string]bool{
		"rep.auction-fetch-state.handling":            true,
		"rep.container-metrics-reporter.tick.started": true,
		"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-run.node-run.monitor-run.run-step.running": true,
		"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-create.starting":                           true,
		"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-completed-container.deleting-container":                                                    true,
		"rep.event-consumer.operation-stream.executing-container-operation.ordinary-lrp-processor.process-reserved-container.run-container.containerstore-run.node-run.action.run-step.running":      true,
		"rep.running-bulker.sync.starting":       true,
		"garden-linux.garden-server.run.spawned": true,
	}

	filteredSignificantEvents := analyzers.SignificantEvents{}

	for name, events := range significantEvents {
		if allow[name] {
			filteredSignificantEvents[name] = events
		}
	}
	filteredSignificantEvents.LogWithThreshold(0.2)

	options := analyzers.SignificantEventsOptions{
		LineOverlays:    gardenStressTestContainerCountOverlays(entries),
		VerticalMarkers: healthCheckTimeoutsFailedHealthMonitor(entries),
		WidthStretch:    2,
		MaxY:            400,
	}

	analyzers.VisualizeSignificantEvents(
		filteredSignificantEvents,
		config.DataDir("health-check-timeouts", "health-check-timeouts.png"),
		options,
	)
}
Beispiel #11
0
func analyzeEventDurations(path string, options analyzers.SignificantEventsOptions, n int, skips []string, outFile string) {
	data, err := ioutil.ReadFile(path)
	say.ExitIfError("couldn't read log file", err)

	entries := util.ChugLagerEntries(data)

	significantEvents := analyzers.ExtractSignificantEventsWithThreshold(entries, n)
	significantEvents.LogWithThreshold(0.2)

	for _, skip := range skips {
		say.Println(0, "Skipping %s", skip)
		delete(significantEvents, skip)
	}

	analyzers.VisualizeSignificantEvents(
		significantEvents,
		outFile,
		options,
	)
}
Beispiel #12
0
func analyzeGardenStressTests() {
	data, err := ioutil.ReadFile(config.DataDir("garden-stress-test", "all.unified"))
	say.ExitIfError("couldn't read log file", err)

	entries := util.ChugLagerEntries(data)

	significantEvents := analyzers.ExtractSignificantEvents(entries)

	allow := map[string]bool{
		"garden-linux.garden-server.create.creating":        true,
		"rep.depot-client.delete-container.destroy.started": true,
	}

	filteredSignificantEvents := analyzers.SignificantEvents{}
	filteredSignificantEvents.LogWithThreshold(0.2)
	for name, events := range significantEvents {
		if allow[name] {
			filteredSignificantEvents[name] = events
		}
	}
	filteredSignificantEvents.LogWithThreshold(0.2)

	limit := viz.NewHorizontalLine(256)
	limit.LineStyle = viz.LineStyle(viz.Red, 1)

	options := analyzers.SignificantEventsOptions{
		LineOverlays:    gardenStressTestContainerCountOverlays(entries),
		VerticalMarkers: gardenStressTestFailedContainerCreates(entries),
		OverlayPlots:    []plot.Plotter{limit},
		WidthStretch:    3,
		MaxY:            400,
	}

	analyzers.VisualizeSignificantEvents(
		filteredSignificantEvents,
		config.DataDir("garden-stress-test", "out.png"),
		options,
	)
}
Beispiel #13
0
func findTheApp() {
	allLrps := map[string][]string{}

	for _, run := range slowPWSTaskRuns {
		say.Println(0, say.Green(run.Name))
		lrps := map[string]bool{}

		data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)
		tMin := run.CliffTimestamp.Add(-2 * time.Minute)
		tCliff := run.CliffTimestamp

		for _, entry := range entries {
			if entry.Timestamp.After(tMin) && entry.Timestamp.Before(tCliff) && entry.Message == "garden-linux.garden-server.bulk_info.got-bulkinfo" {
				handles := reflect.ValueOf(entry.Data["handles"])
				for i := 0; i < handles.Len(); i += 1 {
					handle := handles.Index(i).Interface().(string)
					if len(handle) == 110 {
						guid := handle[0:36]
						lrps[guid] = true
					}
				}
			}
		}
		say.Println(0, format.Object(lrps, 0))

		for lrp := range lrps {
			allLrps[lrp] = append(allLrps[lrp], run.Name)
		}
	}
	say.Println(0, say.Green("Counts"))
	for lrp, runs := range allLrps {
		if len(runs) > 1 {
			say.Println(0, "%s: %s", lrp, say.Green("%s", strings.Join(runs, ", ")))
		}
	}
}
Beispiel #14
0
func NewDescribeCommand() say.Command {
	var name string
	var format string

	flags := flag.NewFlagSet("describe", flag.ContinueOnError)

	flags.StringVar(&name, "name", "", "classroom name")
	flags.StringVar(&format, "format", "json", "output format: json or plain")

	return say.Command{
		Name:        "describe",
		Description: "Describe current state of the classroom",
		FlagSet:     flags,
		Run: func(args []string) {
			validateRequiredArgument("name", name)

			c := newControllerFromEnv()
			output, err := c.DescribeClassroom(name, format)
			say.ExitIfError("Failed describing classroom", err)
			fmt.Println(output)
		},
	}
}
Beispiel #15
0
func NewRunCommand() say.Command {
	var name string
	var command string

	flags := flag.NewFlagSet("run", flag.ContinueOnError)

	flags.StringVar(&name, "name", "", "classroom name")
	flags.StringVar(&command, "c", "", "command to run, parsable by the remote shell")

	return say.Command{
		Name:        "run",
		Description: "Run a command on all VMs, in parallel",
		FlagSet:     flags,
		Run: func(args []string) {
			validateRequiredArgument("name", name)
			validateRequiredArgument("c", command)

			c := newControllerFromEnv()
			err := c.RunOnVMs(name, command)
			say.ExitIfError("Failed running commands in classroom", err)
		},
	}
}
Beispiel #16
0
func NewCreateCommand() say.Command {
	var name string
	var number int

	flags := flag.NewFlagSet("create", flag.ContinueOnError)

	flags.StringVar(&name, "name", "", "classroom name, must be globally unique until destroyed")
	flags.IntVar(&number, "number", 0, "number of VMs to boot")

	return say.Command{
		Name:        "create",
		Description: "Create a fresh classroom environment",
		FlagSet:     flags,
		Run: func(args []string) {
			validateRequiredArgument("name", name)
			validateRequiredArgument("number", number)

			c := newControllerFromEnv()
			err := c.CreateClassroom(name, number)
			say.ExitIfError("Failed creating new classroom", err)
		},
	}
}
Beispiel #17
0
func analyzeGardenDT() {
	data, err := ioutil.ReadFile(config.DataDir("garden-dt", "garden-dt.logs"))
	say.ExitIfError("couldn't read log file", err)

	entries := util.ChugLagerEntries(data)

	significantEvents := analyzers.ExtractSignificantEvents(entries)
	significantEvents.LogWithThreshold(0.2)

	delete(significantEvents, "garden-linux.container.info-starting")

	options := analyzers.SignificantEventsOptions{
		MarkedEvents: map[string]plot.LineStyle{
			"garden-linux.garden-server.create.creating":    viz.LineStyle(viz.Blue, 1, viz.Dot),
			"garden-linux.garden-server.destroy.destroying": viz.LineStyle(viz.Red, 1, viz.Dot),
		},
	}

	analyzers.VisualizeSignificantEvents(
		significantEvents,
		config.DataDir("garden-dt", "many-garden-dt.svg"),
		options,
	)
}
func analyzeAuctioneerFetchStateDuration() {
	data, err := ioutil.ReadFile(config.DataDir("auctioneer-fetch-state-duration", "auctioneer-fetch-state-duration.logs"))
	say.ExitIfError("couldn't read log file", err)

	entries := util.ChugLagerEntries(data)
	fetchStateDurationEvents := ExtractEventsFromLagerData(entries, DurationExtractor("duration"))

	allDurations := fetchStateDurationEvents.Data()
	highDurations := allDurations.Filter(NumFilter(">", 0.2))
	lowDurations := allDurations.Filter(NumFilter("<=", 0.2))

	earlyAllDurations := fetchStateDurationEvents.FilterTimes(NumFilter("<", 1445274027070991039)).Data()
	earlyHighDurations := earlyAllDurations.Filter(NumFilter(">", 0.2))
	earlyLowDurations := earlyAllDurations.Filter(NumFilter("<=", 0.2))

	lateAllDurations := fetchStateDurationEvents.FilterTimes(NumFilter(">=", 1445274027070991039)).Data()
	lateHighDurations := lateAllDurations.Filter(NumFilter(">", 0.2))
	lateLowDurations := lateAllDurations.Filter(NumFilter("<=", 0.2))

	board := viz.NewUniformBoard(3, 1, 0)
	earlyScale := float64(len(allDurations)) / float64(len(earlyAllDurations))
	lateScale := float64(len(allDurations)) / float64(len(lateAllDurations))

	say.Println(0, "All Fetches:   %s", allDurations.Stats())
	say.Println(1, ">  0.2s:     %s", highDurations.Stats())
	say.Println(1, "<= 0.2s:     %s", lowDurations.Stats())
	say.Println(0, "Early Fetches: %s", earlyAllDurations.Stats())
	say.Println(1, ">  0.2s:     %s", earlyHighDurations.Stats())
	say.Println(1, "<= 0.2s:     %s", earlyLowDurations.Stats())
	say.Println(0, "Late Fetches:  %s", lateAllDurations.Stats())
	say.Println(1, ">  0.2s:     %s", lateHighDurations.Stats())
	say.Println(1, "<= 0.2s:     %s", lateLowDurations.Stats())

	p, _ := plot.New()
	p.Title.Text = "All Auctioneer Fetch State Durations"
	p.Add(viz.NewHistogram(allDurations, 20, allDurations.Min(), allDurations.Max()))
	h := viz.NewScaledHistogram(earlyAllDurations, 20, allDurations.Min(), allDurations.Max(), earlyScale)
	h.LineStyle = viz.LineStyle(viz.Blue, 1)
	p.Add(h)
	h = viz.NewScaledHistogram(lateAllDurations, 20, allDurations.Min(), allDurations.Max(), lateScale)
	h.LineStyle = viz.LineStyle(viz.Red, 1)
	p.Add(h)
	board.AddNextSubPlot(p)

	p, _ = plot.New()
	p.Title.Text = "Auctioneer Fetch State Durations &lt; 0.2s"
	p.Add(viz.NewHistogram(lowDurations, 100, lowDurations.Min(), lowDurations.Max()))
	h = viz.NewScaledHistogram(earlyLowDurations, 100, lowDurations.Min(), lowDurations.Max(), earlyScale)
	h.LineStyle = viz.LineStyle(viz.Blue, 1)
	p.Add(h)
	h = viz.NewScaledHistogram(lateLowDurations, 100, lowDurations.Min(), lowDurations.Max(), lateScale)
	h.LineStyle = viz.LineStyle(viz.Red, 1)
	p.Add(h)
	board.AddNextSubPlot(p)

	p, _ = plot.New()
	p.Title.Text = "Auctioneer Fetch State Durations > 0.2s"
	p.Add(viz.NewHistogram(highDurations, 40, highDurations.Min(), highDurations.Max()))
	h = viz.NewScaledHistogram(earlyHighDurations, 40, highDurations.Min(), highDurations.Max(), earlyScale)
	h.LineStyle = viz.LineStyle(viz.Blue, 1)
	p.Add(h)
	h = viz.NewScaledHistogram(lateHighDurations, 40, highDurations.Min(), highDurations.Max(), lateScale)
	h.LineStyle = viz.LineStyle(viz.Red, 1)
	p.Add(h)
	board.AddNextSubPlot(p)

	board.Save(12, 4, config.DataDir("auctioneer-fetch-state-duration", "auctioneer-fetch-state-duration.svg"))
}
func VisualizeSignificantEvents(events SignificantEvents, filename string, options SignificantEventsOptions) {
	firstTime := events.FirstTime()
	tr := func(t time.Time) float64 {
		return t.Sub(firstTime).Seconds()
	}

	minX := options.MinX
	maxX := 0.0
	maxY := 0.0
	minY := math.MaxFloat64

	histograms := map[string]plot.Plotter{}
	scatters := map[string][]plot.Plotter{}
	verticalLines := []plot.Plotter{}

	colorCounter := 0
	for _, name := range events.OrderedNames() {
		xys := plotter.XYs{}
		xErrs := plotter.XErrors{}
		for _, event := range events[name] {
			if event.V > 0 {
				xys = append(xys, struct{ X, Y float64 }{tr(event.T), event.V})
				xErrs = append(xErrs, struct{ Low, High float64 }{-event.V / 2, event.V / 2})

				if tr(event.T) > maxX {
					maxX = tr(event.T)
				}
				if event.V > maxY {
					maxY = event.V
				}
				if event.V < minY {
					minY = event.V
				}
			}
		}

		if len(xys) == 0 {
			say.Println(0, "No data for %s", name)
			continue
		}

		if options.MarkedEvents != nil {
			ls, ok := options.MarkedEvents[name]
			if ok {
				for _, event := range events[name] {
					l := viz.NewVerticalLine(tr(event.T))
					l.LineStyle = ls
					verticalLines = append(verticalLines, l)
				}
			}
		}

		s, err := plotter.NewScatter(xys)
		say.ExitIfError("Couldn't create scatter plot", err)

		s.GlyphStyle = plot.GlyphStyle{
			Color:  viz.OrderedColor(colorCounter),
			Radius: 2,
			Shape:  plot.CircleGlyph{},
		}

		xErrsPlot, err := plotter.NewXErrorBars(struct {
			plotter.XYer
			plotter.XErrorer
		}{xys, xErrs})
		say.ExitIfError("Couldn't create x errors plot", err)
		xErrsPlot.LineStyle = viz.LineStyle(viz.OrderedColor(colorCounter), 1)

		scatters[name] = []plot.Plotter{s, xErrsPlot}

		durations := events[name].Data()
		h := viz.NewHistogram(durations, 20, durations.Min(), durations.Max())
		h.LineStyle = viz.LineStyle(viz.OrderedColor(colorCounter), 1)
		histograms[name] = h
		colorCounter++
	}

	if options.MaxX != 0 {
		maxX = options.MaxX
	}

	maxY = math.Pow(10, math.Ceil(math.Log10(maxY)))
	minY = math.Pow(10, math.Floor(math.Log10(minY)))

	b := &viz.Board{}
	n := len(histograms) + 1
	padding := 0.1 / float64(n-1)
	height := (1.0 - padding*float64(n-1)) / float64(n)
	histWidth := 0.3
	scatterWidth := 0.7
	y := 1 - height - padding - height

	allScatterPlot, _ := plot.New()
	allScatterPlot.Title.Text = "All Events"
	allScatterPlot.X.Label.Text = "Time (s)"
	allScatterPlot.Y.Label.Text = "Duration (s)"
	allScatterPlot.Y.Scale = plot.LogScale
	allScatterPlot.Y.Tick.Marker = plot.LogTicks

	for _, name := range events.OrderedNames() {
		histogram, ok := histograms[name]
		if !ok {
			continue
		}
		scatter := scatters[name]

		allScatterPlot.Add(scatter[0])
		allScatterPlot.Add(scatter[1])

		histogramPlot, _ := plot.New()
		histogramPlot.Title.Text = name
		histogramPlot.X.Label.Text = "Duration (s)"
		histogramPlot.Y.Label.Text = "N"
		histogramPlot.Add(histogram)

		scatterPlot, _ := plot.New()
		scatterPlot.Title.Text = name
		scatterPlot.X.Label.Text = "Time (s)"
		scatterPlot.Y.Label.Text = "Duration (s)"
		scatterPlot.Y.Scale = plot.LogScale
		scatterPlot.Y.Tick.Marker = plot.LogTicks
		scatterPlot.Add(scatter...)
		scatterPlot.Add(verticalLines...)
		scatterPlot.X.Min = minX
		scatterPlot.X.Max = maxX
		scatterPlot.Y.Min = 1e-5
		scatterPlot.Y.Max = maxY

		b.AddSubPlot(histogramPlot, viz.Rect{0, y, histWidth, height})
		b.AddSubPlot(scatterPlot, viz.Rect{histWidth, y, scatterWidth, height})

		y -= height + padding
	}

	allScatterPlot.Add(verticalLines...)
	allScatterPlot.X.Min = minX
	allScatterPlot.X.Max = maxX
	allScatterPlot.Y.Min = 1e-5
	allScatterPlot.Y.Max = maxY
	fmt.Println("all", minX, maxX)

	b.AddSubPlot(allScatterPlot, viz.Rect{histWidth, 1 - height, scatterWidth, height})

	b.Save(16.0, 5*float64(n), filename)
}
func VisualizeSignificantEvents(events SignificantEvents, filename string, options SignificantEventsOptions) {
	firstTime := events.FirstTime()
	tr := func(t time.Time) float64 {
		return t.Sub(firstTime).Seconds()
	}

	minX := 0.0
	if options.MinX != 0 {
		minX = options.MinX
	}
	if !options.MinT.IsZero() {
		minX = tr(options.MinT)
	}
	maxX := 0.0
	maxY := 0.0
	minY := math.MaxFloat64

	scatters := map[string][]plot.Plotter{}
	verticalLines := []plot.Plotter{}
	lineOverlays := []plot.Plotter{}

	colorCounter := 0
	for _, name := range events.OrderedNames() {
		xys := plotter.XYs{}
		xErrs := plotter.XErrors{}
		for _, event := range events[name] {
			if event.V > 0 {
				xys = append(xys, struct{ X, Y float64 }{tr(event.T), event.V})
				xErrs = append(xErrs, struct{ Low, High float64 }{-event.V / 2, event.V / 2})

				if tr(event.T) > maxX {
					maxX = tr(event.T)
				}
				if event.V > maxY {
					maxY = event.V
				}
				if event.V < minY {
					minY = event.V
				}
			}
		}

		if len(xys) == 0 {
			say.Println(0, "No data for %s", name)
			continue
		}

		if options.MarkedEvents != nil {
			ls, ok := options.MarkedEvents[name]
			if ok {
				for _, event := range events[name] {
					l := viz.NewVerticalLine(tr(event.T))
					l.LineStyle = ls
					verticalLines = append(verticalLines, l)
				}
			}
		}

		s, err := plotter.NewScatter(xys)
		say.ExitIfError("Couldn't create scatter plot", err)

		s.GlyphStyle = plot.GlyphStyle{
			Color:  viz.OrderedColor(colorCounter),
			Radius: 2,
			Shape:  plot.CircleGlyph{},
		}

		xErrsPlot, err := plotter.NewXErrorBars(struct {
			plotter.XYer
			plotter.XErrorer
		}{xys, xErrs})
		say.ExitIfError("Couldn't create x errors plot", err)
		xErrsPlot.LineStyle = viz.LineStyle(viz.OrderedColor(colorCounter), 1)

		scatters[name] = []plot.Plotter{s, xErrsPlot}

		colorCounter++
	}

	for _, marker := range options.VerticalMarkers {
		l := viz.NewVerticalLine(tr(marker.T))
		l.LineStyle = marker.LineStyle
		verticalLines = append(verticalLines, l)
	}

	for _, lineOverlay := range options.LineOverlays {
		xys := plotter.XYs{}
		for _, event := range lineOverlay.Events {
			if event.V > 0 {
				xys = append(xys, struct{ X, Y float64 }{tr(event.T), event.V})
			}
		}

		l, s, err := plotter.NewLinePoints(xys)
		say.ExitIfError("Couldn't create scatter plot", err)

		l.LineStyle = lineOverlay.LineStyle
		s.GlyphStyle = plot.GlyphStyle{
			Color:  lineOverlay.LineStyle.Color,
			Radius: lineOverlay.LineStyle.Width,
			Shape:  plot.CrossGlyph{},
		}
		lineOverlays = append(lineOverlays, l, s)
	}

	if options.MaxX != 0 {
		maxX = options.MaxX
	}
	if !options.MaxT.IsZero() {
		maxX = tr(options.MaxT)
	}

	maxY = math.Pow(10, math.Ceil(math.Log10(maxY)))

	if options.MaxY != 0 {
		maxY = options.MaxY
	}

	minY = math.Pow(10, math.Floor(math.Log10(minY)))

	n := len(scatters) + 1
	b := viz.NewUniformBoard(1, n, 0.01)

	allScatterPlot, _ := plot.New()
	allScatterPlot.Title.Text = "All Events"
	allScatterPlot.X.Label.Text = "Time (s)"
	allScatterPlot.Y.Label.Text = "Duration (s)"
	allScatterPlot.Y.Scale = plot.LogScale
	allScatterPlot.Y.Tick.Marker = plot.LogTicks

	for i, name := range events.OrderedNames() {
		scatter, ok := scatters[name]
		if !ok {
			continue
		}

		allScatterPlot.Add(scatter[0])
		allScatterPlot.Add(scatter[1])

		scatterPlot, _ := plot.New()
		scatterPlot.Title.Text = name
		scatterPlot.X.Label.Text = "Time (s)"
		scatterPlot.Y.Label.Text = "Duration (s)"
		scatterPlot.Y.Scale = plot.LogScale
		scatterPlot.Y.Tick.Marker = plot.LogTicks
		scatterPlot.Add(scatter...)
		scatterPlot.Add(verticalLines...)
		scatterPlot.Add(lineOverlays...)
		scatterPlot.Add(options.OverlayPlots...)
		scatterPlot.X.Min = minX
		scatterPlot.X.Max = maxX
		scatterPlot.Y.Min = 1e-5
		scatterPlot.Y.Max = maxY

		b.AddSubPlotAt(scatterPlot, 0, n-2-i)
	}

	allScatterPlot.Add(verticalLines...)
	allScatterPlot.Add(lineOverlays...)
	allScatterPlot.Add(options.OverlayPlots...)
	allScatterPlot.X.Min = minX
	allScatterPlot.X.Max = maxX
	allScatterPlot.Y.Min = 1e-5
	allScatterPlot.Y.Max = maxY
	fmt.Println("all", minX, maxX)

	b.AddSubPlotAt(allScatterPlot, 0, n-1)

	width := 12.0
	if options.WidthStretch > 0 {
		width = width * options.WidthStretch
	}
	b.Save(width, 5*float64(n), filename)
}