func usage(commandGroups []common.CommandGroup) {
if len(os.Args) > 2 {
matcher := strings.ToLower(os.Args[2])
for _, commandGroup := range commandGroups {
if strings.HasPrefix(strings.ToLower(commandGroup.Name), matcher) {
usageForCommandGroup(commandGroup, true)
return
}
for _, command := range commandGroup.Commands {
if strings.HasPrefix(strings.ToLower(command.Name), matcher) {
usageForCommand(0, command, true)
return
}
}
}
say.Fprintln(os.Stderr, 0, say.Red("Unkown command: %s", os.Args[2]))
}
say.Fprintln(os.Stderr, 0, "%s", say.Cyan("Help and Autocompletion"))
say.Fprintln(os.Stderr, 0, strings.Repeat("-", len("Help and Autocompletion")))
say.Fprintln(os.Stderr, 1, "%s %s", say.Green("help"), say.LightGray("[command] - Show this help, or detailed help for the passed in command"))
say.Fprintln(os.Stderr, 1, "%s %s", say.Green("completions"), say.LightGray("Generate BASH Completions for veritas"))
say.Fprintln(os.Stderr, 0, "")
for _, commandGroup := range commandGroups {
usageForCommandGroup(commandGroup, false)
say.Println(0, "")
}
}
func printDomains(domains []string) {
say.Println(0, say.Green("Domains"))
if len(domains) == 0 {
say.Println(1, say.Red("None"))
return
}
for _, domain := range domains {
say.Println(1, say.Green("%s", domain))
}
}
func colorByTaskSuccess(task *models.Task, format string, args ...interface{}) string {
if task.Failed {
return say.Red(format, args...)
} else {
return say.Green(format, args...)
}
}
func printTasks(verbose bool, tasks veritas_models.VeritasTasks) {
taskTypes := tasks.OrderedTaskTypes()
say.Println(0, say.Green("Tasks"))
for _, taskType := range taskTypes {
say.Println(0, say.Green(taskType))
for _, task := range tasks[taskType] {
if verbose {
printVerboseTask(task)
} else {
printTask(task)
}
}
}
}
func ServeLogs(addr string, dev bool, minTime time.Time, maxTime time.Time, match *regexp.Regexp, exclude *regexp.Regexp, src io.Reader) error {
out := ChugWithFilter(src, minTime, maxTime, match, exclude)
entries := []JSFriendlyChugEntry{}
for entry := range out {
if isEmptyInigoLog(entry) {
continue
}
jsEntry := NewJSFriendlyChugEntry(entry)
entries = append(entries, jsEntry)
}
http.HandleFunc("/assets/", AssetServer(dev))
http.HandleFunc("/data", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(entries)
})
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/assets/index.html", http.StatusTemporaryRedirect)
})
listener, err := net.Listen("tcp", addr)
say.Println(0, say.Green("Serving up on http://127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port))
if err != nil {
return err
}
return http.Serve(listener, nil)
}
func routes(info *models.Routes) string {
if info == nil {
return ""
}
data, found := (*info)[CF_ROUTER]
if !found || data == nil {
return ""
}
routes := CFRoutes{}
err := json.Unmarshal(*data, &routes)
if err != nil {
return ""
}
out := ""
for _, route := range routes {
out += fmt.Sprintf("%s => %s ", say.Yellow("%d", route.Port), say.Green(strings.Join(route.Hostnames, " ")))
}
return out
}
func dumpVitals(client *http.Client, component string, addr string, out io.Writer) {
response, err := client.Get("http://" + addr + "/debug/pprof/")
if err != nil {
say.Println(0, say.Red("%s: %s"), component, err.Error())
return
}
if response.StatusCode != http.StatusOK {
say.Println(0, say.Red("%s: %d"), component, response.StatusCode)
return
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
say.Println(0, say.Red("%s: %d"), component, err.Error())
return
}
s := string(body)
report := []string{}
rows := strings.Split(s, "<tr>")[1:]
for _, row := range rows {
columns := strings.Split(row, "<td>")
value, _ := strconv.Atoi(strings.Split(columns[0], ">")[1])
name := strings.Split(columns[1], ">")[1]
name = name[:len(name)-3]
if value > 1000 {
report = append(report, say.Red("%20s", fmt.Sprintf("%d %s", value, name)))
} else {
report = append(report, fmt.Sprintf("%20s", fmt.Sprintf("%d %s", value, name)))
}
}
say.Println(0, "%s: %s %s", say.Green("%25s", component), string(strings.Join(report, " ")), addr)
}
func RepState(out io.Writer) (err error) {
client := rep.NewClient(&http.Client{
Timeout: 5 * time.Second,
}, "http://localhost:1800")
t := time.Now()
state, err := client.State()
dt := time.Since(t)
if err != nil {
say.Println(0, "Cell State [%s] - Error:%s", dt, say.Red(err.Error()))
return err
}
name := say.Green("Cell State")
if state.Evacuating {
name = say.Red("Cell State - EVAC -")
}
rootFSes := []string{}
for key := range state.RootFSProviders {
if key != "preloaded" {
rootFSes = append(rootFSes, say.Yellow(key))
}
}
for key := range state.RootFSProviders["preloaded"].(rep.FixedSetRootFSProvider).FixedSet {
rootFSes = append(rootFSes, say.Green("preloaded:%s", key))
}
say.Println(0, "%s [%s] - Zone:%s | %s Tasks, %s LRPs | C:%d/%d M:%d/%d D:%d/%d | %s",
name,
dt,
say.Cyan(state.Zone),
say.Cyan("%d", len(state.Tasks)),
say.Cyan("%d", len(state.LRPs)),
state.AvailableResources.Containers,
state.TotalResources.Containers,
state.AvailableResources.MemoryMB,
state.TotalResources.MemoryMB,
state.AvailableResources.DiskMB,
state.TotalResources.DiskMB,
strings.Join(rootFSes, ", "),
)
return nil
}
func usage(commands []Command) {
say.FprintBanner(os.Stderr, "Probe", "=")
for _, command := range commands {
say.Fprintln(os.Stderr, 0, "%s %s", say.Green(command.Name), say.LightGray(command.Description))
command.FlagSet.PrintDefaults()
say.Fprintln(os.Stderr, 0, "")
}
}
func usageForCommand(indentation int, command common.Command, includeFlags bool) {
say.Fprintln(os.Stderr, indentation, "%s %s", say.Green(command.Name), say.LightGray(command.Description))
if includeFlags {
buffer := &bytes.Buffer{}
command.FlagSet.SetOutput(buffer)
command.FlagSet.PrintDefaults()
say.Fprintln(os.Stderr, indentation, buffer.String())
}
}
func monitor(route string, pollInterval, batchInterval time.Duration) {
say.Println(0, "Monitoring %s every %s", say.Green(route), say.Green("%s", pollInterval))
// http.DefaultClient.Timeout = 200 * time.Millisecond
ticker := time.NewTicker(pollInterval)
startTime := time.Now()
roundTime := time.Now()
indices := map[int]int{}
requests := 0
succesfulRequests := 0
for {
<-ticker.C
requests++
resp, err := http.Get(route + "/index")
if err != nil {
say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Error: %s", err.Error())))
continue
}
if resp.StatusCode != http.StatusOK {
say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Invalid Status Code: %d", resp.StatusCode)))
say.Println(1, say.Red(format.Object(resp.Header, 0)))
continue
}
succesfulRequests++
indexStr, _ := ioutil.ReadAll(resp.Body)
index, _ := strconv.Atoi(string(indexStr))
indices[index]++
resp.Body.Close()
if time.Since(roundTime) >= batchInterval {
say.Println(0, "%s: %d/%d %s", say.Yellow("%s", time.Since(startTime)), succesfulRequests, requests, sortedIndices(indices))
indices = map[int]int{}
requests = 0
succesfulRequests = 0
roundTime = time.Now()
}
}
}
func printLRPS(verbose bool, lrps veritas_models.VeritasLRPS) {
say.Println(0, say.Green("LRPs"))
sortedLRPS := lrps.SortedByProcessGuid()
for _, lrp := range sortedLRPS {
if verbose {
printVerboseLRP(lrp)
} else {
printLRP(lrp)
}
}
}
func findTheApp() {
allLrps := map[string][]string{}
for _, run := range slowPWSTaskRuns {
say.Println(0, say.Green(run.Name))
lrps := map[string]bool{}
data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
tMin := run.CliffTimestamp.Add(-2 * time.Minute)
tCliff := run.CliffTimestamp
for _, entry := range entries {
if entry.Timestamp.After(tMin) && entry.Timestamp.Before(tCliff) && entry.Message == "garden-linux.garden-server.bulk_info.got-bulkinfo" {
handles := reflect.ValueOf(entry.Data["handles"])
for i := 0; i < handles.Len(); i += 1 {
handle := handles.Index(i).Interface().(string)
if len(handle) == 110 {
guid := handle[0:36]
lrps[guid] = true
}
}
}
}
say.Println(0, format.Object(lrps, 0))
for lrp := range lrps {
allLrps[lrp] = append(allLrps[lrp], run.Name)
}
}
say.Println(0, say.Green("Counts"))
for lrp, runs := range allLrps {
if len(runs) > 1 {
say.Println(0, "%s: %s", lrp, say.Green("%s", strings.Join(runs, ", ")))
}
}
}
func analyzeSlowPWSTasks() {
for _, run := range slowPWSTaskRuns {
say.Println(0, say.Green(run.Name))
data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
allow := map[string]bool{
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.depot-client.run-container.creating-container-in-garden": true,
"rep.depot-client.delete-container.destroy.started": true,
"rep.depot-client.run-container.run.action.download-step.fetch-starting": true,
"rep.depot-client.run-container.run.monitor-run.run-step.running": true,
"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
"rep.depot-client.run-container.run.setup.download-step.fetch-starting": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
options := analyzers.SignificantEventsOptions{
LineOverlays: containerCountOverlays(entries),
}
if !run.CliffTimestamp.IsZero() {
options.MaxT = run.EndTimestamp.Add(time.Minute * 30)
options.VerticalMarkers = []analyzers.VerticalMarker{
{T: run.CliffTimestamp, LineStyle: viz.LineStyle(viz.Red, 1, viz.Dash)},
{T: run.EndTimestamp, LineStyle: viz.LineStyle(viz.Black, 1, viz.Dash)},
}
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("pws-slow-tasks", run.Name+".png"),
options,
)
}
}
func analyzeCPUWeightStresstest() {
runs := []string{
"unmodified-run",
"aufs-run",
"2-conc-run",
}
for _, run := range runs {
say.Println(0, say.Green(run))
data, err := ioutil.ReadFile(config.DataDir("cpu-wait-stress-test", run+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
allow := map[string]bool{
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.depot-client.run-container.creating-container-in-garden": true,
"rep.depot-client.delete-container.destroy.started": true,
"rep.depot-client.run-container.run.action.download-step.fetch-starting": true,
"rep.depot-client.run-container.run.monitor-run.run-step.running": true,
"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
"rep.depot-client.run-container.run.setup.download-step.fetch-starting": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
options := analyzers.SignificantEventsOptions{
LineOverlays: cpuWeightStressTestContainerCountOverlays(entries),
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("cpu-wait-stress-test", run+".png"),
options,
)
}
}
func actualState(actual *models.ActualLRP) string {
switch actual.State {
case models.ActualLRPStateUnclaimed:
if actual.PlacementError == "" {
return say.LightGray("UNCLAIMED")
} else {
return say.Red("UNCLAIMED (%s)", actual.PlacementError)
}
case models.ActualLRPStateClaimed:
return say.Yellow("CLAIMED")
case models.ActualLRPStateRunning:
return say.Green("RUNNING")
case models.ActualLRPStateCrashed:
return say.Red("CRASHED (%d - %s)", actual.CrashCount, strings.Replace(actual.CrashReason, "\n", " ", -1))
default:
return say.Red("INVALID")
}
}
func SetDomain(bbsClient bbs.Client, domain string, ttl time.Duration) error {
say.Println(0, say.Green("Setting Domain %s with TTL %ds", domain, int(ttl.Seconds())))
return bbsClient.UpsertDomain(domain, ttl)
}
func printContainer(out io.Writer, containerInfo ContainerInfo) {
info := containerInfo.Info
metrics := containerInfo.Metrics
say.Fprintln(out, 0,
"%s - %s @ %s",
say.Green(containerInfo.Handle),
info.State,
info.ContainerPath,
)
say.Fprintln(out, 1,
"Memory: %.3f MB",
float64(metrics.MemoryStat.TotalRss+metrics.MemoryStat.TotalCache-metrics.MemoryStat.TotalInactiveFile)/1024.0/1024.0,
)
say.Fprintln(out, 1,
"Disk: Total:%.3f MB %d Inodes, Exclusive:%.3f MB %d Inodes",
float64(metrics.DiskStat.TotalBytesUsed)/1024.0/1024.0,
metrics.DiskStat.TotalInodesUsed,
float64(metrics.DiskStat.ExclusiveBytesUsed)/1024.0/1024.0,
metrics.DiskStat.ExclusiveInodesUsed,
)
ports := []string{}
for _, portMapping := range info.MappedPorts {
ports = append(ports, fmt.Sprintf("%d:%d", portMapping.HostPort, portMapping.ContainerPort))
}
say.Fprintln(out, 1,
"%s=>%s: %s",
say.Green(info.HostIP),
say.Green(containerInfo.Handle),
strings.Join(ports, ","),
)
if len(info.Events) > 0 {
say.Fprintln(out, 1,
"Events: %s",
strings.Join(info.Events, ","),
)
}
if len(info.ProcessIDs) > 0 {
say.Fprintln(out, 1,
"Running: %d processes",
len(info.ProcessIDs),
)
}
if len(info.Properties) > 0 {
say.Fprintln(out, 1,
"Properties:",
)
for key, value := range info.Properties {
say.Fprintln(out, 2,
"%s: %s",
key, value,
)
}
}
}
func (l *CliLogger) Green(f string, a ...interface{}) string {
return say.Green(f, a...)
}
func printLRP(lrp *veritas_models.VeritasLRP) {
say.Println(1, say.Green(lrp.ProcessGuid))
if lrp.DesiredLRP.GetProcessGuid() != "" {
privileged := ""
if lrp.DesiredLRP.Privileged {
privileged = say.Red(" PRIVILEGED")
}
routesString := routes(lrp.DesiredLRP.Routes)
if routesString != "" {
routesString = "\n" + say.Indent(1, routesString)
}
say.Println(
2,
"%s %s%s (%d MB, %d MB, %d CPU)%s",
say.Green("%d", lrp.DesiredLRP.Instances),
say.Cyan(lrp.DesiredLRP.RootFs),
privileged,
lrp.DesiredLRP.MemoryMb,
lrp.DesiredLRP.DiskMb,
lrp.DesiredLRP.CpuWeight,
routesString,
)
} else {
say.Println(2, say.Red("UNDESIRED"))
}
orderedActualIndices := lrp.OrderedActualLRPIndices()
for _, index := range orderedActualIndices {
actualLRPGroup := lrp.ActualLRPGroupsByIndex[index]
if instance := actualLRPGroup.Instance; instance != nil {
if instance.State == models.ActualLRPStateUnclaimed || instance.State == models.ActualLRPStateCrashed {
say.Println(
3,
"%2s: [%s for %s]",
index,
actualState(instance),
time.Since(time.Unix(0, instance.Since)),
)
} else {
say.Println(
3,
"%2s: %s %s [%s for %s]",
index,
instance.InstanceGuid,
say.Yellow(instance.CellId),
actualState(instance),
time.Since(time.Unix(0, instance.Since)),
)
}
}
if evacuating := actualLRPGroup.Evacuating; evacuating != nil {
say.Println(
3,
"%s: %s %s [%s for %s] - %s",
say.Red("%2s", index),
say.Red(evacuating.InstanceGuid),
say.Yellow(evacuating.CellId),
actualState(evacuating),
time.Since(time.Unix(0, evacuating.Since)),
say.Red("EVACUATING"),
)
}
}
}