Beispiel #1
0
func usage(commandGroups []common.CommandGroup) {
	if len(os.Args) > 2 {
		matcher := strings.ToLower(os.Args[2])
		for _, commandGroup := range commandGroups {
			if strings.HasPrefix(strings.ToLower(commandGroup.Name), matcher) {
				usageForCommandGroup(commandGroup, true)
				return
			}

			for _, command := range commandGroup.Commands {
				if strings.HasPrefix(strings.ToLower(command.Name), matcher) {
					usageForCommand(0, command, true)
					return
				}
			}
		}
		say.Fprintln(os.Stderr, 0, say.Red("Unkown command: %s", os.Args[2]))
	}

	say.Fprintln(os.Stderr, 0, "%s", say.Cyan("Help and Autocompletion"))
	say.Fprintln(os.Stderr, 0, strings.Repeat("-", len("Help and Autocompletion")))
	say.Fprintln(os.Stderr, 1, "%s %s", say.Green("help"), say.LightGray("[command] - Show this help, or detailed help for the passed in command"))
	say.Fprintln(os.Stderr, 1, "%s %s", say.Green("completions"), say.LightGray("Generate BASH Completions for veritas"))
	say.Fprintln(os.Stderr, 0, "")

	for _, commandGroup := range commandGroups {
		usageForCommandGroup(commandGroup, false)
		say.Println(0, "")
	}
}
Beispiel #2
0
func printDomains(domains []string) {
	say.Println(0, say.Green("Domains"))
	if len(domains) == 0 {
		say.Println(1, say.Red("None"))
		return
	}
	for _, domain := range domains {
		say.Println(1, say.Green("%s", domain))
	}
}
Beispiel #3
0
func colorByTaskSuccess(task *models.Task, format string, args ...interface{}) string {
	if task.Failed {
		return say.Red(format, args...)
	} else {
		return say.Green(format, args...)
	}
}
Beispiel #4
0
func printTasks(verbose bool, tasks veritas_models.VeritasTasks) {
	taskTypes := tasks.OrderedTaskTypes()

	say.Println(0, say.Green("Tasks"))

	for _, taskType := range taskTypes {
		say.Println(0, say.Green(taskType))
		for _, task := range tasks[taskType] {
			if verbose {
				printVerboseTask(task)
			} else {
				printTask(task)
			}
		}
	}
}
Beispiel #5
0
func ServeLogs(addr string, dev bool, minTime time.Time, maxTime time.Time, match *regexp.Regexp, exclude *regexp.Regexp, src io.Reader) error {
	out := ChugWithFilter(src, minTime, maxTime, match, exclude)

	entries := []JSFriendlyChugEntry{}
	for entry := range out {
		if isEmptyInigoLog(entry) {
			continue
		}
		jsEntry := NewJSFriendlyChugEntry(entry)
		entries = append(entries, jsEntry)
	}

	http.HandleFunc("/assets/", AssetServer(dev))

	http.HandleFunc("/data", func(w http.ResponseWriter, r *http.Request) {
		json.NewEncoder(w).Encode(entries)
	})

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/assets/index.html", http.StatusTemporaryRedirect)
	})

	listener, err := net.Listen("tcp", addr)
	say.Println(0, say.Green("Serving up on http://127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port))
	if err != nil {
		return err
	}

	return http.Serve(listener, nil)
}
Beispiel #6
0
func routes(info *models.Routes) string {
	if info == nil {
		return ""
	}

	data, found := (*info)[CF_ROUTER]
	if !found || data == nil {
		return ""
	}

	routes := CFRoutes{}
	err := json.Unmarshal(*data, &routes)

	if err != nil {
		return ""
	}

	out := ""

	for _, route := range routes {
		out += fmt.Sprintf("%s => %s ", say.Yellow("%d", route.Port), say.Green(strings.Join(route.Hostnames, " ")))
	}

	return out
}
Beispiel #7
0
func dumpVitals(client *http.Client, component string, addr string, out io.Writer) {
	response, err := client.Get("http://" + addr + "/debug/pprof/")
	if err != nil {
		say.Println(0, say.Red("%s: %s"), component, err.Error())
		return
	}
	if response.StatusCode != http.StatusOK {
		say.Println(0, say.Red("%s: %d"), component, response.StatusCode)
		return
	}

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		say.Println(0, say.Red("%s: %d"), component, err.Error())
		return
	}

	s := string(body)
	report := []string{}
	rows := strings.Split(s, "<tr>")[1:]
	for _, row := range rows {
		columns := strings.Split(row, "<td>")
		value, _ := strconv.Atoi(strings.Split(columns[0], ">")[1])
		name := strings.Split(columns[1], ">")[1]
		name = name[:len(name)-3]
		if value > 1000 {
			report = append(report, say.Red("%20s", fmt.Sprintf("%d %s", value, name)))
		} else {
			report = append(report, fmt.Sprintf("%20s", fmt.Sprintf("%d %s", value, name)))
		}
	}

	say.Println(0, "%s: %s %s", say.Green("%25s", component), string(strings.Join(report, " ")), addr)
}
Beispiel #8
0
func RepState(out io.Writer) (err error) {
	client := rep.NewClient(&http.Client{
		Timeout: 5 * time.Second,
	}, "http://localhost:1800")

	t := time.Now()
	state, err := client.State()
	dt := time.Since(t)

	if err != nil {
		say.Println(0, "Cell State [%s] - Error:%s", dt, say.Red(err.Error()))
		return err
	}

	name := say.Green("Cell State")
	if state.Evacuating {
		name = say.Red("Cell State - EVAC -")
	}

	rootFSes := []string{}
	for key := range state.RootFSProviders {
		if key != "preloaded" {
			rootFSes = append(rootFSes, say.Yellow(key))
		}
	}

	for key := range state.RootFSProviders["preloaded"].(rep.FixedSetRootFSProvider).FixedSet {
		rootFSes = append(rootFSes, say.Green("preloaded:%s", key))
	}

	say.Println(0, "%s [%s] - Zone:%s | %s Tasks, %s LRPs | C:%d/%d M:%d/%d D:%d/%d | %s",
		name,
		dt,
		say.Cyan(state.Zone),
		say.Cyan("%d", len(state.Tasks)),
		say.Cyan("%d", len(state.LRPs)),
		state.AvailableResources.Containers,
		state.TotalResources.Containers,
		state.AvailableResources.MemoryMB,
		state.TotalResources.MemoryMB,
		state.AvailableResources.DiskMB,
		state.TotalResources.DiskMB,
		strings.Join(rootFSes, ", "),
	)

	return nil
}
Beispiel #9
0
func usage(commands []Command) {
	say.FprintBanner(os.Stderr, "Probe", "=")
	for _, command := range commands {
		say.Fprintln(os.Stderr, 0, "%s %s", say.Green(command.Name), say.LightGray(command.Description))
		command.FlagSet.PrintDefaults()
		say.Fprintln(os.Stderr, 0, "")
	}
}
Beispiel #10
0
func usageForCommand(indentation int, command common.Command, includeFlags bool) {
	say.Fprintln(os.Stderr, indentation, "%s %s", say.Green(command.Name), say.LightGray(command.Description))
	if includeFlags {
		buffer := &bytes.Buffer{}
		command.FlagSet.SetOutput(buffer)
		command.FlagSet.PrintDefaults()
		say.Fprintln(os.Stderr, indentation, buffer.String())
	}
}
Beispiel #11
0
func monitor(route string, pollInterval, batchInterval time.Duration) {
	say.Println(0, "Monitoring %s every %s", say.Green(route), say.Green("%s", pollInterval))
	// http.DefaultClient.Timeout = 200 * time.Millisecond

	ticker := time.NewTicker(pollInterval)

	startTime := time.Now()
	roundTime := time.Now()
	indices := map[int]int{}
	requests := 0
	succesfulRequests := 0
	for {
		<-ticker.C
		requests++
		resp, err := http.Get(route + "/index")

		if err != nil {
			say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Error: %s", err.Error())))
			continue
		}

		if resp.StatusCode != http.StatusOK {
			say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Invalid Status Code: %d", resp.StatusCode)))
			say.Println(1, say.Red(format.Object(resp.Header, 0)))
			continue
		}

		succesfulRequests++
		indexStr, _ := ioutil.ReadAll(resp.Body)
		index, _ := strconv.Atoi(string(indexStr))

		indices[index]++
		resp.Body.Close()

		if time.Since(roundTime) >= batchInterval {
			say.Println(0, "%s: %d/%d %s", say.Yellow("%s", time.Since(startTime)), succesfulRequests, requests, sortedIndices(indices))
			indices = map[int]int{}
			requests = 0
			succesfulRequests = 0
			roundTime = time.Now()
		}
	}
}
Beispiel #12
0
func printLRPS(verbose bool, lrps veritas_models.VeritasLRPS) {
	say.Println(0, say.Green("LRPs"))

	sortedLRPS := lrps.SortedByProcessGuid()
	for _, lrp := range sortedLRPS {
		if verbose {
			printVerboseLRP(lrp)
		} else {
			printLRP(lrp)
		}
	}
}
Beispiel #13
0
func findTheApp() {
	allLrps := map[string][]string{}

	for _, run := range slowPWSTaskRuns {
		say.Println(0, say.Green(run.Name))
		lrps := map[string]bool{}

		data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)
		tMin := run.CliffTimestamp.Add(-2 * time.Minute)
		tCliff := run.CliffTimestamp

		for _, entry := range entries {
			if entry.Timestamp.After(tMin) && entry.Timestamp.Before(tCliff) && entry.Message == "garden-linux.garden-server.bulk_info.got-bulkinfo" {
				handles := reflect.ValueOf(entry.Data["handles"])
				for i := 0; i < handles.Len(); i += 1 {
					handle := handles.Index(i).Interface().(string)
					if len(handle) == 110 {
						guid := handle[0:36]
						lrps[guid] = true
					}
				}
			}
		}
		say.Println(0, format.Object(lrps, 0))

		for lrp := range lrps {
			allLrps[lrp] = append(allLrps[lrp], run.Name)
		}
	}
	say.Println(0, say.Green("Counts"))
	for lrp, runs := range allLrps {
		if len(runs) > 1 {
			say.Println(0, "%s: %s", lrp, say.Green("%s", strings.Join(runs, ", ")))
		}
	}
}
Beispiel #14
0
func analyzeSlowPWSTasks() {
	for _, run := range slowPWSTaskRuns {
		say.Println(0, say.Green(run.Name))
		data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)

		allow := map[string]bool{
			"rep.auction-fetch-state.handling":                                             true,
			"rep.container-metrics-reporter.tick.started":                                  true,
			"rep.depot-client.run-container.creating-container-in-garden":                  true,
			"rep.depot-client.delete-container.destroy.started":                            true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":       true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":              true,
			"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":        true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		options := analyzers.SignificantEventsOptions{
			LineOverlays: containerCountOverlays(entries),
		}
		if !run.CliffTimestamp.IsZero() {
			options.MaxT = run.EndTimestamp.Add(time.Minute * 30)
			options.VerticalMarkers = []analyzers.VerticalMarker{
				{T: run.CliffTimestamp, LineStyle: viz.LineStyle(viz.Red, 1, viz.Dash)},
				{T: run.EndTimestamp, LineStyle: viz.LineStyle(viz.Black, 1, viz.Dash)},
			}
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("pws-slow-tasks", run.Name+".png"),
			options,
		)

	}
}
Beispiel #15
0
func analyzeCPUWeightStresstest() {
	runs := []string{
		"unmodified-run",
		"aufs-run",
		"2-conc-run",
	}
	for _, run := range runs {
		say.Println(0, say.Green(run))
		data, err := ioutil.ReadFile(config.DataDir("cpu-wait-stress-test", run+".unified"))
		say.ExitIfError("couldn't read log file", err)

		entries := util.ChugLagerEntries(data)

		significantEvents := analyzers.ExtractSignificantEvents(entries)

		allow := map[string]bool{
			"rep.auction-fetch-state.handling":                                             true,
			"rep.container-metrics-reporter.tick.started":                                  true,
			"rep.depot-client.run-container.creating-container-in-garden":                  true,
			"rep.depot-client.delete-container.destroy.started":                            true,
			"rep.depot-client.run-container.run.action.download-step.fetch-starting":       true,
			"rep.depot-client.run-container.run.monitor-run.run-step.running":              true,
			"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
			"rep.depot-client.run-container.run.setup.download-step.fetch-starting":        true,
		}

		filteredSignificantEvents := analyzers.SignificantEvents{}
		for name, events := range significantEvents {
			if allow[name] {
				filteredSignificantEvents[name] = events
			}
		}
		filteredSignificantEvents.LogWithThreshold(0.2)

		options := analyzers.SignificantEventsOptions{
			LineOverlays: cpuWeightStressTestContainerCountOverlays(entries),
		}

		analyzers.VisualizeSignificantEvents(
			filteredSignificantEvents,
			config.DataDir("cpu-wait-stress-test", run+".png"),
			options,
		)

	}
}
Beispiel #16
0
func actualState(actual *models.ActualLRP) string {
	switch actual.State {
	case models.ActualLRPStateUnclaimed:
		if actual.PlacementError == "" {
			return say.LightGray("UNCLAIMED")
		} else {
			return say.Red("UNCLAIMED (%s)", actual.PlacementError)
		}
	case models.ActualLRPStateClaimed:
		return say.Yellow("CLAIMED")
	case models.ActualLRPStateRunning:
		return say.Green("RUNNING")
	case models.ActualLRPStateCrashed:
		return say.Red("CRASHED (%d - %s)", actual.CrashCount, strings.Replace(actual.CrashReason, "\n", " ", -1))
	default:
		return say.Red("INVALID")
	}
}
Beispiel #17
0
func SetDomain(bbsClient bbs.Client, domain string, ttl time.Duration) error {
	say.Println(0, say.Green("Setting Domain %s with TTL %ds", domain, int(ttl.Seconds())))

	return bbsClient.UpsertDomain(domain, ttl)
}
Beispiel #18
0
func printContainer(out io.Writer, containerInfo ContainerInfo) {
	info := containerInfo.Info
	metrics := containerInfo.Metrics
	say.Fprintln(out, 0,
		"%s - %s @ %s",
		say.Green(containerInfo.Handle),
		info.State,
		info.ContainerPath,
	)

	say.Fprintln(out, 1,
		"Memory: %.3f MB",
		float64(metrics.MemoryStat.TotalRss+metrics.MemoryStat.TotalCache-metrics.MemoryStat.TotalInactiveFile)/1024.0/1024.0,
	)

	say.Fprintln(out, 1,
		"Disk: Total:%.3f MB %d Inodes, Exclusive:%.3f MB %d Inodes",
		float64(metrics.DiskStat.TotalBytesUsed)/1024.0/1024.0,
		metrics.DiskStat.TotalInodesUsed,
		float64(metrics.DiskStat.ExclusiveBytesUsed)/1024.0/1024.0,
		metrics.DiskStat.ExclusiveInodesUsed,
	)

	ports := []string{}
	for _, portMapping := range info.MappedPorts {
		ports = append(ports, fmt.Sprintf("%d:%d", portMapping.HostPort, portMapping.ContainerPort))
	}

	say.Fprintln(out, 1,
		"%s=>%s: %s",
		say.Green(info.HostIP),
		say.Green(containerInfo.Handle),
		strings.Join(ports, ","),
	)

	if len(info.Events) > 0 {
		say.Fprintln(out, 1,
			"Events: %s",
			strings.Join(info.Events, ","),
		)
	}

	if len(info.ProcessIDs) > 0 {
		say.Fprintln(out, 1,
			"Running: %d processes",
			len(info.ProcessIDs),
		)
	}

	if len(info.Properties) > 0 {
		say.Fprintln(out, 1,
			"Properties:",
		)
		for key, value := range info.Properties {
			say.Fprintln(out, 2,
				"%s: %s",
				key, value,
			)
		}
	}
}
Beispiel #19
0
func (l *CliLogger) Green(f string, a ...interface{}) string {
	return say.Green(f, a...)
}
Beispiel #20
0
func printLRP(lrp *veritas_models.VeritasLRP) {
	say.Println(1, say.Green(lrp.ProcessGuid))
	if lrp.DesiredLRP.GetProcessGuid() != "" {
		privileged := ""
		if lrp.DesiredLRP.Privileged {
			privileged = say.Red(" PRIVILEGED")
		}

		routesString := routes(lrp.DesiredLRP.Routes)
		if routesString != "" {
			routesString = "\n" + say.Indent(1, routesString)
		}

		say.Println(
			2,
			"%s %s%s (%d MB, %d MB, %d CPU)%s",
			say.Green("%d", lrp.DesiredLRP.Instances),
			say.Cyan(lrp.DesiredLRP.RootFs),
			privileged,
			lrp.DesiredLRP.MemoryMb,
			lrp.DesiredLRP.DiskMb,
			lrp.DesiredLRP.CpuWeight,
			routesString,
		)
	} else {
		say.Println(2, say.Red("UNDESIRED"))
	}

	orderedActualIndices := lrp.OrderedActualLRPIndices()
	for _, index := range orderedActualIndices {
		actualLRPGroup := lrp.ActualLRPGroupsByIndex[index]
		if instance := actualLRPGroup.Instance; instance != nil {
			if instance.State == models.ActualLRPStateUnclaimed || instance.State == models.ActualLRPStateCrashed {
				say.Println(
					3,
					"%2s: [%s for %s]",
					index,
					actualState(instance),
					time.Since(time.Unix(0, instance.Since)),
				)
			} else {
				say.Println(
					3,
					"%2s: %s %s [%s for %s]",
					index,
					instance.InstanceGuid,
					say.Yellow(instance.CellId),
					actualState(instance),
					time.Since(time.Unix(0, instance.Since)),
				)
			}
		}
		if evacuating := actualLRPGroup.Evacuating; evacuating != nil {
			say.Println(
				3,
				"%s: %s %s [%s for %s] - %s",
				say.Red("%2s", index),
				say.Red(evacuating.InstanceGuid),
				say.Yellow(evacuating.CellId),
				actualState(evacuating),
				time.Since(time.Unix(0, evacuating.Since)),
				say.Red("EVACUATING"),
			)
		}
	}
}