func usage(commandGroups []common.CommandGroup) { if len(os.Args) > 2 { matcher := strings.ToLower(os.Args[2]) for _, commandGroup := range commandGroups { if strings.HasPrefix(strings.ToLower(commandGroup.Name), matcher) { usageForCommandGroup(commandGroup, true) return } for _, command := range commandGroup.Commands { if strings.HasPrefix(strings.ToLower(command.Name), matcher) { usageForCommand(0, command, true) return } } } say.Fprintln(os.Stderr, 0, say.Red("Unkown command: %s", os.Args[2])) } say.Fprintln(os.Stderr, 0, "%s", say.Cyan("Help and Autocompletion")) say.Fprintln(os.Stderr, 0, strings.Repeat("-", len("Help and Autocompletion"))) say.Fprintln(os.Stderr, 1, "%s %s", say.Green("help"), say.LightGray("[command] - Show this help, or detailed help for the passed in command")) say.Fprintln(os.Stderr, 1, "%s %s", say.Green("completions"), say.LightGray("Generate BASH Completions for veritas")) say.Fprintln(os.Stderr, 0, "") for _, commandGroup := range commandGroups { usageForCommandGroup(commandGroup, false) say.Println(0, "") } }
func printDomains(domains []string) { say.Println(0, say.Green("Domains")) if len(domains) == 0 { say.Println(1, say.Red("None")) return } for _, domain := range domains { say.Println(1, say.Green("%s", domain)) } }
func colorByTaskSuccess(task *models.Task, format string, args ...interface{}) string { if task.Failed { return say.Red(format, args...) } else { return say.Green(format, args...) } }
func printTasks(verbose bool, tasks veritas_models.VeritasTasks) { taskTypes := tasks.OrderedTaskTypes() say.Println(0, say.Green("Tasks")) for _, taskType := range taskTypes { say.Println(0, say.Green(taskType)) for _, task := range tasks[taskType] { if verbose { printVerboseTask(task) } else { printTask(task) } } } }
func ServeLogs(addr string, dev bool, minTime time.Time, maxTime time.Time, match *regexp.Regexp, exclude *regexp.Regexp, src io.Reader) error { out := ChugWithFilter(src, minTime, maxTime, match, exclude) entries := []JSFriendlyChugEntry{} for entry := range out { if isEmptyInigoLog(entry) { continue } jsEntry := NewJSFriendlyChugEntry(entry) entries = append(entries, jsEntry) } http.HandleFunc("/assets/", AssetServer(dev)) http.HandleFunc("/data", func(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(entries) }) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/assets/index.html", http.StatusTemporaryRedirect) }) listener, err := net.Listen("tcp", addr) say.Println(0, say.Green("Serving up on http://127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port)) if err != nil { return err } return http.Serve(listener, nil) }
func routes(info *models.Routes) string { if info == nil { return "" } data, found := (*info)[CF_ROUTER] if !found || data == nil { return "" } routes := CFRoutes{} err := json.Unmarshal(*data, &routes) if err != nil { return "" } out := "" for _, route := range routes { out += fmt.Sprintf("%s => %s ", say.Yellow("%d", route.Port), say.Green(strings.Join(route.Hostnames, " "))) } return out }
func dumpVitals(client *http.Client, component string, addr string, out io.Writer) { response, err := client.Get("http://" + addr + "/debug/pprof/") if err != nil { say.Println(0, say.Red("%s: %s"), component, err.Error()) return } if response.StatusCode != http.StatusOK { say.Println(0, say.Red("%s: %d"), component, response.StatusCode) return } body, err := ioutil.ReadAll(response.Body) if err != nil { say.Println(0, say.Red("%s: %d"), component, err.Error()) return } s := string(body) report := []string{} rows := strings.Split(s, "<tr>")[1:] for _, row := range rows { columns := strings.Split(row, "<td>") value, _ := strconv.Atoi(strings.Split(columns[0], ">")[1]) name := strings.Split(columns[1], ">")[1] name = name[:len(name)-3] if value > 1000 { report = append(report, say.Red("%20s", fmt.Sprintf("%d %s", value, name))) } else { report = append(report, fmt.Sprintf("%20s", fmt.Sprintf("%d %s", value, name))) } } say.Println(0, "%s: %s %s", say.Green("%25s", component), string(strings.Join(report, " ")), addr) }
func RepState(out io.Writer) (err error) { client := rep.NewClient(&http.Client{ Timeout: 5 * time.Second, }, "http://localhost:1800") t := time.Now() state, err := client.State() dt := time.Since(t) if err != nil { say.Println(0, "Cell State [%s] - Error:%s", dt, say.Red(err.Error())) return err } name := say.Green("Cell State") if state.Evacuating { name = say.Red("Cell State - EVAC -") } rootFSes := []string{} for key := range state.RootFSProviders { if key != "preloaded" { rootFSes = append(rootFSes, say.Yellow(key)) } } for key := range state.RootFSProviders["preloaded"].(rep.FixedSetRootFSProvider).FixedSet { rootFSes = append(rootFSes, say.Green("preloaded:%s", key)) } say.Println(0, "%s [%s] - Zone:%s | %s Tasks, %s LRPs | C:%d/%d M:%d/%d D:%d/%d | %s", name, dt, say.Cyan(state.Zone), say.Cyan("%d", len(state.Tasks)), say.Cyan("%d", len(state.LRPs)), state.AvailableResources.Containers, state.TotalResources.Containers, state.AvailableResources.MemoryMB, state.TotalResources.MemoryMB, state.AvailableResources.DiskMB, state.TotalResources.DiskMB, strings.Join(rootFSes, ", "), ) return nil }
func usage(commands []Command) { say.FprintBanner(os.Stderr, "Probe", "=") for _, command := range commands { say.Fprintln(os.Stderr, 0, "%s %s", say.Green(command.Name), say.LightGray(command.Description)) command.FlagSet.PrintDefaults() say.Fprintln(os.Stderr, 0, "") } }
func usageForCommand(indentation int, command common.Command, includeFlags bool) { say.Fprintln(os.Stderr, indentation, "%s %s", say.Green(command.Name), say.LightGray(command.Description)) if includeFlags { buffer := &bytes.Buffer{} command.FlagSet.SetOutput(buffer) command.FlagSet.PrintDefaults() say.Fprintln(os.Stderr, indentation, buffer.String()) } }
func monitor(route string, pollInterval, batchInterval time.Duration) { say.Println(0, "Monitoring %s every %s", say.Green(route), say.Green("%s", pollInterval)) // http.DefaultClient.Timeout = 200 * time.Millisecond ticker := time.NewTicker(pollInterval) startTime := time.Now() roundTime := time.Now() indices := map[int]int{} requests := 0 succesfulRequests := 0 for { <-ticker.C requests++ resp, err := http.Get(route + "/index") if err != nil { say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Error: %s", err.Error()))) continue } if resp.StatusCode != http.StatusOK { say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Invalid Status Code: %d", resp.StatusCode))) say.Println(1, say.Red(format.Object(resp.Header, 0))) continue } succesfulRequests++ indexStr, _ := ioutil.ReadAll(resp.Body) index, _ := strconv.Atoi(string(indexStr)) indices[index]++ resp.Body.Close() if time.Since(roundTime) >= batchInterval { say.Println(0, "%s: %d/%d %s", say.Yellow("%s", time.Since(startTime)), succesfulRequests, requests, sortedIndices(indices)) indices = map[int]int{} requests = 0 succesfulRequests = 0 roundTime = time.Now() } } }
func printLRPS(verbose bool, lrps veritas_models.VeritasLRPS) { say.Println(0, say.Green("LRPs")) sortedLRPS := lrps.SortedByProcessGuid() for _, lrp := range sortedLRPS { if verbose { printVerboseLRP(lrp) } else { printLRP(lrp) } } }
func findTheApp() { allLrps := map[string][]string{} for _, run := range slowPWSTaskRuns { say.Println(0, say.Green(run.Name)) lrps := map[string]bool{} data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified")) say.ExitIfError("couldn't read log file", err) entries := util.ChugLagerEntries(data) tMin := run.CliffTimestamp.Add(-2 * time.Minute) tCliff := run.CliffTimestamp for _, entry := range entries { if entry.Timestamp.After(tMin) && entry.Timestamp.Before(tCliff) && entry.Message == "garden-linux.garden-server.bulk_info.got-bulkinfo" { handles := reflect.ValueOf(entry.Data["handles"]) for i := 0; i < handles.Len(); i += 1 { handle := handles.Index(i).Interface().(string) if len(handle) == 110 { guid := handle[0:36] lrps[guid] = true } } } } say.Println(0, format.Object(lrps, 0)) for lrp := range lrps { allLrps[lrp] = append(allLrps[lrp], run.Name) } } say.Println(0, say.Green("Counts")) for lrp, runs := range allLrps { if len(runs) > 1 { say.Println(0, "%s: %s", lrp, say.Green("%s", strings.Join(runs, ", "))) } } }
func analyzeSlowPWSTasks() { for _, run := range slowPWSTaskRuns { say.Println(0, say.Green(run.Name)) data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified")) say.ExitIfError("couldn't read log file", err) entries := util.ChugLagerEntries(data) significantEvents := analyzers.ExtractSignificantEvents(entries) allow := map[string]bool{ "rep.auction-fetch-state.handling": true, "rep.container-metrics-reporter.tick.started": true, "rep.depot-client.run-container.creating-container-in-garden": true, "rep.depot-client.delete-container.destroy.started": true, "rep.depot-client.run-container.run.action.download-step.fetch-starting": true, "rep.depot-client.run-container.run.monitor-run.run-step.running": true, "rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true, "rep.depot-client.run-container.run.setup.download-step.fetch-starting": true, } filteredSignificantEvents := analyzers.SignificantEvents{} for name, events := range significantEvents { if allow[name] { filteredSignificantEvents[name] = events } } filteredSignificantEvents.LogWithThreshold(0.2) options := analyzers.SignificantEventsOptions{ LineOverlays: containerCountOverlays(entries), } if !run.CliffTimestamp.IsZero() { options.MaxT = run.EndTimestamp.Add(time.Minute * 30) options.VerticalMarkers = []analyzers.VerticalMarker{ {T: run.CliffTimestamp, LineStyle: viz.LineStyle(viz.Red, 1, viz.Dash)}, {T: run.EndTimestamp, LineStyle: viz.LineStyle(viz.Black, 1, viz.Dash)}, } } analyzers.VisualizeSignificantEvents( filteredSignificantEvents, config.DataDir("pws-slow-tasks", run.Name+".png"), options, ) } }
func analyzeCPUWeightStresstest() { runs := []string{ "unmodified-run", "aufs-run", "2-conc-run", } for _, run := range runs { say.Println(0, say.Green(run)) data, err := ioutil.ReadFile(config.DataDir("cpu-wait-stress-test", run+".unified")) say.ExitIfError("couldn't read log file", err) entries := util.ChugLagerEntries(data) significantEvents := analyzers.ExtractSignificantEvents(entries) allow := map[string]bool{ "rep.auction-fetch-state.handling": true, "rep.container-metrics-reporter.tick.started": true, "rep.depot-client.run-container.creating-container-in-garden": true, "rep.depot-client.delete-container.destroy.started": true, "rep.depot-client.run-container.run.action.download-step.fetch-starting": true, "rep.depot-client.run-container.run.monitor-run.run-step.running": true, "rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true, "rep.depot-client.run-container.run.setup.download-step.fetch-starting": true, } filteredSignificantEvents := analyzers.SignificantEvents{} for name, events := range significantEvents { if allow[name] { filteredSignificantEvents[name] = events } } filteredSignificantEvents.LogWithThreshold(0.2) options := analyzers.SignificantEventsOptions{ LineOverlays: cpuWeightStressTestContainerCountOverlays(entries), } analyzers.VisualizeSignificantEvents( filteredSignificantEvents, config.DataDir("cpu-wait-stress-test", run+".png"), options, ) } }
func actualState(actual *models.ActualLRP) string { switch actual.State { case models.ActualLRPStateUnclaimed: if actual.PlacementError == "" { return say.LightGray("UNCLAIMED") } else { return say.Red("UNCLAIMED (%s)", actual.PlacementError) } case models.ActualLRPStateClaimed: return say.Yellow("CLAIMED") case models.ActualLRPStateRunning: return say.Green("RUNNING") case models.ActualLRPStateCrashed: return say.Red("CRASHED (%d - %s)", actual.CrashCount, strings.Replace(actual.CrashReason, "\n", " ", -1)) default: return say.Red("INVALID") } }
func SetDomain(bbsClient bbs.Client, domain string, ttl time.Duration) error { say.Println(0, say.Green("Setting Domain %s with TTL %ds", domain, int(ttl.Seconds()))) return bbsClient.UpsertDomain(domain, ttl) }
func printContainer(out io.Writer, containerInfo ContainerInfo) { info := containerInfo.Info metrics := containerInfo.Metrics say.Fprintln(out, 0, "%s - %s @ %s", say.Green(containerInfo.Handle), info.State, info.ContainerPath, ) say.Fprintln(out, 1, "Memory: %.3f MB", float64(metrics.MemoryStat.TotalRss+metrics.MemoryStat.TotalCache-metrics.MemoryStat.TotalInactiveFile)/1024.0/1024.0, ) say.Fprintln(out, 1, "Disk: Total:%.3f MB %d Inodes, Exclusive:%.3f MB %d Inodes", float64(metrics.DiskStat.TotalBytesUsed)/1024.0/1024.0, metrics.DiskStat.TotalInodesUsed, float64(metrics.DiskStat.ExclusiveBytesUsed)/1024.0/1024.0, metrics.DiskStat.ExclusiveInodesUsed, ) ports := []string{} for _, portMapping := range info.MappedPorts { ports = append(ports, fmt.Sprintf("%d:%d", portMapping.HostPort, portMapping.ContainerPort)) } say.Fprintln(out, 1, "%s=>%s: %s", say.Green(info.HostIP), say.Green(containerInfo.Handle), strings.Join(ports, ","), ) if len(info.Events) > 0 { say.Fprintln(out, 1, "Events: %s", strings.Join(info.Events, ","), ) } if len(info.ProcessIDs) > 0 { say.Fprintln(out, 1, "Running: %d processes", len(info.ProcessIDs), ) } if len(info.Properties) > 0 { say.Fprintln(out, 1, "Properties:", ) for key, value := range info.Properties { say.Fprintln(out, 2, "%s: %s", key, value, ) } } }
func (l *CliLogger) Green(f string, a ...interface{}) string { return say.Green(f, a...) }
func printLRP(lrp *veritas_models.VeritasLRP) { say.Println(1, say.Green(lrp.ProcessGuid)) if lrp.DesiredLRP.GetProcessGuid() != "" { privileged := "" if lrp.DesiredLRP.Privileged { privileged = say.Red(" PRIVILEGED") } routesString := routes(lrp.DesiredLRP.Routes) if routesString != "" { routesString = "\n" + say.Indent(1, routesString) } say.Println( 2, "%s %s%s (%d MB, %d MB, %d CPU)%s", say.Green("%d", lrp.DesiredLRP.Instances), say.Cyan(lrp.DesiredLRP.RootFs), privileged, lrp.DesiredLRP.MemoryMb, lrp.DesiredLRP.DiskMb, lrp.DesiredLRP.CpuWeight, routesString, ) } else { say.Println(2, say.Red("UNDESIRED")) } orderedActualIndices := lrp.OrderedActualLRPIndices() for _, index := range orderedActualIndices { actualLRPGroup := lrp.ActualLRPGroupsByIndex[index] if instance := actualLRPGroup.Instance; instance != nil { if instance.State == models.ActualLRPStateUnclaimed || instance.State == models.ActualLRPStateCrashed { say.Println( 3, "%2s: [%s for %s]", index, actualState(instance), time.Since(time.Unix(0, instance.Since)), ) } else { say.Println( 3, "%2s: %s %s [%s for %s]", index, instance.InstanceGuid, say.Yellow(instance.CellId), actualState(instance), time.Since(time.Unix(0, instance.Since)), ) } } if evacuating := actualLRPGroup.Evacuating; evacuating != nil { say.Println( 3, "%s: %s %s [%s for %s] - %s", say.Red("%2s", index), say.Red(evacuating.InstanceGuid), say.Yellow(evacuating.CellId), actualState(evacuating), time.Since(time.Unix(0, evacuating.Since)), say.Red("EVACUATING"), ) } } }