func dumpVitals(client *http.Client, component string, addr string, out io.Writer) {
response, err := client.Get("http://" + addr + "/debug/pprof/")
if err != nil {
say.Println(0, say.Red("%s: %s"), component, err.Error())
return
}
if response.StatusCode != http.StatusOK {
say.Println(0, say.Red("%s: %d"), component, response.StatusCode)
return
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
say.Println(0, say.Red("%s: %d"), component, err.Error())
return
}
s := string(body)
report := []string{}
rows := strings.Split(s, "<tr>")[1:]
for _, row := range rows {
columns := strings.Split(row, "<td>")
value, _ := strconv.Atoi(strings.Split(columns[0], ">")[1])
name := strings.Split(columns[1], ">")[1]
name = name[:len(name)-3]
if value > 1000 {
report = append(report, say.Red("%20s", fmt.Sprintf("%d %s", value, name)))
} else {
report = append(report, fmt.Sprintf("%20s", fmt.Sprintf("%d %s", value, name)))
}
}
say.Println(0, "%s: %s %s", say.Green("%25s", component), string(strings.Join(report, " ")), addr)
}
func ExtractSignificantEvents(entries []chug.LogEntry) SignificantEvents {
say.Println(0, "Log Lines: %d", len(entries))
say.Println(0, "Spanning: %s", entries[len(entries)-1].Timestamp.Sub(entries[0].Timestamp))
bySession := map[string][]chug.LogEntry{}
for _, entry := range entries {
session := entry.Source + "-" + entry.Session
bySession[session] = append(bySession[session], entry)
}
say.Println(0, "Sessions: %d", len(bySession))
groupedEvents := map[string]Events{}
for _, entries := range bySession {
duration := entries[len(entries)-1].Timestamp.Sub(entries[0].Timestamp)
timestamp := entries[0].Timestamp.Add(duration / 2)
message := entries[0].Message
groupedEvents[message] = append(groupedEvents[message], Event{T: timestamp, V: duration.Seconds()})
}
say.Println(0, "Grouped Events: %d", len(groupedEvents))
significantEvents := SignificantEvents{}
messages := []string{}
for message, events := range groupedEvents {
if len(events) > 1 {
significantEvents[message] = events
messages = append(messages, message)
}
}
say.Println(0, "Significant Events: %d", len(significantEvents))
return significantEvents
}
func printDomains(domains []string) {
say.Println(0, say.Green("Domains"))
if len(domains) == 0 {
say.Println(1, say.Red("None"))
return
}
for _, domain := range domains {
say.Println(1, say.Green("%s", domain))
}
}
func (s *stenographer) PrettyPrintLog(log chug.LogEntry) {
components := []string{}
color, ok := colorLookup[strings.Split(log.Source, ":")[0]]
if !ok {
color = say.DefaultStyle
}
level := ""
switch log.LogLevel {
case lager.INFO:
level = say.Colorize(color, "%-7s", "[INFO]")
case lager.DEBUG:
level = say.Gray("%-7s", "[DEBUG]")
case lager.ERROR:
level = say.Red("%-7s", "[ERROR]")
case lager.FATAL:
level = say.Red("%-7s", "[FATAL]")
}
var timestamp string
if s.Absolute {
timestamp = log.Timestamp.Format("01/02 15:04:05.00")
} else {
timestamp = log.Timestamp.Sub(s.RelativeTime).String()
timestamp = fmt.Sprintf("%17s", timestamp)
}
components = append(components, say.Colorize(color, "%-16s", log.Source))
components = append(components, level)
components = append(components, say.Colorize(color, timestamp))
components = append(components, say.Gray("%-10s", log.Session))
components = append(components, say.Colorize(color, log.Message))
say.Println(0, strings.Join(components, " "))
if log.Error != nil {
say.Println(27, say.Red("Error: %s", log.Error.Error()))
}
if log.Trace != "" {
say.Println(27, say.Red(log.Trace))
}
if len(log.Data) > 0 && s.Data == "short" {
dataJSON, _ := json.Marshal(log.Data)
say.Println(27, string(dataJSON))
}
if len(log.Data) > 0 && s.Data == "long" {
dataJSON, _ := json.MarshalIndent(log.Data, "", " ")
say.Println(27, string(dataJSON))
}
}
func GetActualLRPCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("get-actual-lrp", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "get-actual-lrp",
Description: "<process-guid> <optional: index> - get an ActualLRP",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
var index = -1
if len(args) == 0 {
say.Fprintln(os.Stderr, 0, say.Red("missing process-guid"))
os.Exit(1)
}
processGuid := args[0]
if len(args) == 2 {
index, err = strconv.Atoi(args[1])
common.ExitIfError("Could not parse index", err)
}
if index == -1 {
actualLRPGroups, err := bbsClient.ActualLRPGroupsByProcessGuid(processGuid)
common.ExitIfError("Could not fetch ActualLRPs", err)
for _, actualLRPGroup := range actualLRPGroups {
actualLRP, _ := actualLRPGroup.Resolve()
preview, _ := json.MarshalIndent(actualLRP, "", " ")
say.Println(0, string(preview))
}
} else {
actualLRPGroup, err := bbsClient.ActualLRPGroupByProcessGuidAndIndex(processGuid, index)
common.ExitIfError("Could not fetch ActualLRP", err)
actualLRP, _ := actualLRPGroup.Resolve()
preview, _ := json.MarshalIndent(actualLRP, "", " ")
say.Println(0, string(preview))
}
},
}
}
func ServeLogs(addr string, dev bool, minTime time.Time, maxTime time.Time, match *regexp.Regexp, exclude *regexp.Regexp, src io.Reader) error {
out := ChugWithFilter(src, minTime, maxTime, match, exclude)
entries := []JSFriendlyChugEntry{}
for entry := range out {
if isEmptyInigoLog(entry) {
continue
}
jsEntry := NewJSFriendlyChugEntry(entry)
entries = append(entries, jsEntry)
}
http.HandleFunc("/assets/", AssetServer(dev))
http.HandleFunc("/data", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(entries)
})
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/assets/index.html", http.StatusTemporaryRedirect)
})
listener, err := net.Listen("tcp", addr)
say.Println(0, say.Green("Serving up on http://127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port))
if err != nil {
return err
}
return http.Serve(listener, nil)
}
func GetDesiredLRPCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("get-desired-lrp", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "get-desired-lrp",
Description: "<process-guid> - get a DesiredLRP",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
if len(args) == 0 {
say.Fprintln(os.Stderr, 0, say.Red("missing process-guid"))
os.Exit(1)
}
desiredLRP, err := bbsClient.DesiredLRPByProcessGuid(args[0])
common.ExitIfError("Failed to fetch DesiredLRP", err)
preview, _ := json.MarshalIndent(desiredLRP, "", " ")
say.Println(0, string(preview))
},
}
}
func completions(commandGroups []common.CommandGroup) {
availableCommands := []string{}
for _, commands := range commandGroups {
for _, command := range commands.Commands {
availableCommands = append(availableCommands, command.Name)
}
}
out := fmt.Sprintf(`
function _veritas() {
local cur prev commands
COMPREPLY=()
cur="${COMP_WORDS[COMP_CWORD]}"
prev="${COMP_WORDS[COMP_CWORD-1]}"
commands="%s"
if [[ "${COMP_CWORD}" == "1" ]] ; then
COMPREPLY=( $(compgen -W "${commands} help completions" -- ${cur}) );
elif [[ "${prev}" == "help" ]] ; then
COMPREPLY=( $(compgen -W "${commands}" -- ${cur}) );
else
COMPREPLY=( $(compgen -f ${cur}) );
fi
return 0
}
complete -F _veritas veritas
`, strings.Join(availableCommands, " "))
say.Println(0, out)
}
func printTasks(verbose bool, tasks veritas_models.VeritasTasks) {
taskTypes := tasks.OrderedTaskTypes()
say.Println(0, say.Green("Tasks"))
for _, taskType := range taskTypes {
say.Println(0, say.Green(taskType))
for _, task := range tasks[taskType] {
if verbose {
printVerboseTask(task)
} else {
printTask(task)
}
}
}
}
func usage(commandGroups []common.CommandGroup) {
if len(os.Args) > 2 {
matcher := strings.ToLower(os.Args[2])
for _, commandGroup := range commandGroups {
if strings.HasPrefix(strings.ToLower(commandGroup.Name), matcher) {
usageForCommandGroup(commandGroup, true)
return
}
for _, command := range commandGroup.Commands {
if strings.HasPrefix(strings.ToLower(command.Name), matcher) {
usageForCommand(0, command, true)
return
}
}
}
say.Fprintln(os.Stderr, 0, say.Red("Unkown command: %s", os.Args[2]))
}
say.Fprintln(os.Stderr, 0, "%s", say.Cyan("Help and Autocompletion"))
say.Fprintln(os.Stderr, 0, strings.Repeat("-", len("Help and Autocompletion")))
say.Fprintln(os.Stderr, 1, "%s %s", say.Green("help"), say.LightGray("[command] - Show this help, or detailed help for the passed in command"))
say.Fprintln(os.Stderr, 1, "%s %s", say.Green("completions"), say.LightGray("Generate BASH Completions for veritas"))
say.Fprintln(os.Stderr, 0, "")
for _, commandGroup := range commandGroups {
usageForCommandGroup(commandGroup, false)
say.Println(0, "")
}
}
func UnifyChugCommand() common.Command {
var (
minTimeFlag string
maxTimeFlag string
matchFlag string
excludeFlag string
)
flagSet := baseFlagSet("chug-unify", &minTimeFlag, &maxTimeFlag, &matchFlag, &excludeFlag)
return common.Command{
Name: "chug-unify",
Description: "file1, file2,... - Combine lager files in temporal order",
FlagSet: flagSet,
Run: func(args []string) {
minTime, maxTime, match, exclude := parseBaseFlags(minTimeFlag, maxTimeFlag, matchFlag, excludeFlag)
if len(args) == 0 {
say.Println(0, say.Red("You must pass chug-unify files to combine"))
os.Exit(1)
} else {
files := []io.Reader{}
for _, arg := range args {
f, err := os.Open(arg)
common.ExitIfError("Could not open file", err)
files = append(files, f)
}
err := Unify(files, os.Stdout, minTime, maxTime, match, exclude)
common.ExitIfError("Failed to chug-unify", err)
}
},
}
}
func RepState(out io.Writer) (err error) {
client := rep.NewClient(&http.Client{
Timeout: 5 * time.Second,
}, "http://localhost:1800")
t := time.Now()
state, err := client.State()
dt := time.Since(t)
if err != nil {
say.Println(0, "Cell State [%s] - Error:%s", dt, say.Red(err.Error()))
return err
}
name := say.Green("Cell State")
if state.Evacuating {
name = say.Red("Cell State - EVAC -")
}
rootFSes := []string{}
for key := range state.RootFSProviders {
if key != "preloaded" {
rootFSes = append(rootFSes, say.Yellow(key))
}
}
for key := range state.RootFSProviders["preloaded"].(rep.FixedSetRootFSProvider).FixedSet {
rootFSes = append(rootFSes, say.Green("preloaded:%s", key))
}
say.Println(0, "%s [%s] - Zone:%s | %s Tasks, %s LRPs | C:%d/%d M:%d/%d D:%d/%d | %s",
name,
dt,
say.Cyan(state.Zone),
say.Cyan("%d", len(state.Tasks)),
say.Cyan("%d", len(state.LRPs)),
state.AvailableResources.Containers,
state.TotalResources.Containers,
state.AvailableResources.MemoryMB,
state.TotalResources.MemoryMB,
state.AvailableResources.DiskMB,
state.TotalResources.DiskMB,
strings.Join(rootFSes, ", "),
)
return nil
}
func TestProctor(t *testing.T) {
if os.Getenv("SKIP_AWS_TESTS") == "true" {
say.Println(0, say.Yellow("WARNING: Skipping AWS integration suite"))
return
}
rand.Seed(config.GinkgoConfig.RandomSeed)
RegisterFailHandler(Fail)
RunSpecs(t, "AWS Integration Suite")
}
func UpdateDesiredLRPCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("update-lrp", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "update-lrp",
Description: "<process-guid> <path to json file> - update a DesiredLRP",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
var raw = []byte{}
if len(args) == 0 {
say.Fprintln(os.Stderr, 0, say.Red("missing process-guid"))
os.Exit(1)
} else if len(args) == 1 {
say.Fprintln(os.Stderr, 0, "Reading from stdin...")
raw, err = ioutil.ReadAll(os.Stdin)
common.ExitIfError("Failed to read from stdin", err)
} else {
raw, err = ioutil.ReadFile(args[1])
common.ExitIfError("Failed to read from file", err)
}
desiredLRPUpdate := &models.DesiredLRPUpdate{}
err = json.Unmarshal(raw, desiredLRPUpdate)
common.ExitIfError("Failed to decode DesiredLRPUpdate", err)
say.Println(0, "Updating %s:", args[0])
preview, _ := json.MarshalIndent(desiredLRPUpdate, "", " ")
say.Println(0, string(preview))
err = bbsClient.UpdateDesiredLRP(args[0], desiredLRPUpdate)
common.ExitIfError("Failed to update DesiredLRP", err)
},
}
}
func monitor(route string, pollInterval, batchInterval time.Duration) {
say.Println(0, "Monitoring %s every %s", say.Green(route), say.Green("%s", pollInterval))
// http.DefaultClient.Timeout = 200 * time.Millisecond
ticker := time.NewTicker(pollInterval)
startTime := time.Now()
roundTime := time.Now()
indices := map[int]int{}
requests := 0
succesfulRequests := 0
for {
<-ticker.C
requests++
resp, err := http.Get(route + "/index")
if err != nil {
say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Error: %s", err.Error())))
continue
}
if resp.StatusCode != http.StatusOK {
say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Invalid Status Code: %d", resp.StatusCode)))
say.Println(1, say.Red(format.Object(resp.Header, 0)))
continue
}
succesfulRequests++
indexStr, _ := ioutil.ReadAll(resp.Body)
index, _ := strconv.Atoi(string(indexStr))
indices[index]++
resp.Body.Close()
if time.Since(roundTime) >= batchInterval {
say.Println(0, "%s: %d/%d %s", say.Yellow("%s", time.Since(startTime)), succesfulRequests, requests, sortedIndices(indices))
indices = map[int]int{}
requests = 0
succesfulRequests = 0
roundTime = time.Now()
}
}
}
func (s SignificantEvents) LogWithThreshold(threshold float64) {
for _, message := range s.OrderedNames() {
events := s[message]
s := fmt.Sprintf("%s %s", events.Data().Stats(), message)
if events.Data().Max() > threshold {
s = say.Red(s)
}
say.Println(0, s)
}
}
func CreateDesiredLRPCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("desire-lrp", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "desire-lrp",
Description: "<path to json file> - create a DesiredLRP",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
var raw = []byte{}
if len(args) == 0 {
say.Fprintln(os.Stderr, 0, "Reading from stdin...")
raw, err = ioutil.ReadAll(os.Stdin)
common.ExitIfError("Failed to read from stdin", err)
} else {
raw, err = ioutil.ReadFile(args[0])
common.ExitIfError("Failed to read from file", err)
}
desiredLRP := &models.DesiredLRP{}
err = json.Unmarshal(raw, desiredLRP)
common.ExitIfError("Failed to decode DesiredLRP", err)
say.Println(0, "Desiring:")
preview, _ := json.MarshalIndent(desiredLRP, "", " ")
say.Println(0, string(preview))
err = bbsClient.DesireLRP(desiredLRP)
common.ExitIfError("Failed to desire DesiredLRP", err)
},
}
}
func printLRPS(verbose bool, lrps veritas_models.VeritasLRPS) {
say.Println(0, say.Green("LRPs"))
sortedLRPS := lrps.SortedByProcessGuid()
for _, lrp := range sortedLRPS {
if verbose {
printVerboseLRP(lrp)
} else {
printLRP(lrp)
}
}
}
//Save saves the board (i.e. all subplots, appropriately laid out) to the specified filename.
//It basically rips off the implementation of Save in plotinum to support various file formats.
func (b *Board) Save(width, height float64, file string) (err error) {
w, h := vg.Inches(width), vg.Inches(height)
var c interface {
vg.Canvas
Size() (w, h vg.Length)
io.WriterTo
}
switch ext := strings.ToLower(filepath.Ext(file)); ext {
case ".eps":
c = vgeps.NewTitle(w, h, file)
case ".jpg", ".jpeg":
c = vgimg.JpegCanvas{Canvas: vgimg.New(w, h)}
case ".pdf":
c = vgpdf.New(w, h)
case ".png":
c = vgimg.PngCanvas{Canvas: vgimg.New(w, h)}
case ".svg":
c = vgsvg.New(w, h)
case ".tiff":
c = vgimg.TiffCanvas{Canvas: vgimg.New(w, h)}
default:
return fmt.Errorf("Unsupported file extension: %s", ext)
}
for _, subplot := range b.SubPlots {
w, h := c.Size()
drawArea := plot.DrawArea{
Canvas: c,
Rect: subplot.ScaledRect(float64(w), float64(h)),
}
subplot.Plot.Draw(drawArea)
}
f, err := os.Create(file)
if err != nil {
return err
}
if _, err = c.WriteTo(f); err != nil {
return err
}
say.Println(0, say.Yellow("Saved %s", file))
return f.Close()
}
func findTheApp() {
allLrps := map[string][]string{}
for _, run := range slowPWSTaskRuns {
say.Println(0, say.Green(run.Name))
lrps := map[string]bool{}
data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
tMin := run.CliffTimestamp.Add(-2 * time.Minute)
tCliff := run.CliffTimestamp
for _, entry := range entries {
if entry.Timestamp.After(tMin) && entry.Timestamp.Before(tCliff) && entry.Message == "garden-linux.garden-server.bulk_info.got-bulkinfo" {
handles := reflect.ValueOf(entry.Data["handles"])
for i := 0; i < handles.Len(); i += 1 {
handle := handles.Index(i).Interface().(string)
if len(handle) == 110 {
guid := handle[0:36]
lrps[guid] = true
}
}
}
}
say.Println(0, format.Object(lrps, 0))
for lrp := range lrps {
allLrps[lrp] = append(allLrps[lrp], run.Name)
}
}
say.Println(0, say.Green("Counts"))
for lrp, runs := range allLrps {
if len(runs) > 1 {
say.Println(0, "%s: %s", lrp, say.Green("%s", strings.Join(runs, ", ")))
}
}
}
func GenerateEventDurationCommand() say.Command {
var minT, maxT float64
var skipList string
var blueEvent, redEvent string
var outFile string
var significantThreshold int
var fs = &flag.FlagSet{}
fs.Float64Var(&minT, "tmin", 0, "Min time")
fs.Float64Var(&maxT, "tmax", 0, "Max time")
fs.StringVar(&skipList, "skip", "", "Events to skip (comma delimited)")
fs.StringVar(&blueEvent, "blue", "", "Events to use to generate blue markers")
fs.StringVar(&redEvent, "red", "", "Events to use to generate red markers")
fs.IntVar(&significantThreshold, "n", 2, "Minimum number of events required to make it onto the plot")
fs.StringVar(&outFile, "o", "", "Output file")
return say.Command{
Name: "event-duration",
Description: "lager.log -- generate event duration plots",
FlagSet: fs,
Run: func(args []string) {
if len(args) != 1 {
say.Println(0, say.Red("please provide a lager file to read"))
os.Exit(1)
}
markedEvents := map[string]plot.LineStyle{}
if blueEvent != "" {
markedEvents[blueEvent] = viz.LineStyle(viz.Blue, 1, viz.Dot)
}
if redEvent != "" {
markedEvents[redEvent] = viz.LineStyle(viz.Red, 1, viz.Dot)
}
options := analyzers.SignificantEventsOptions{
MinX: minT,
MaxX: maxT,
MarkedEvents: markedEvents,
}
skips := strings.Split(skipList, ",")
if outFile == "" {
outFile = "out.png"
}
analyzeEventDurations(args[0], options, significantThreshold, skips, outFile)
},
}
}
func analyzeSlowPWSTasks() {
for _, run := range slowPWSTaskRuns {
say.Println(0, say.Green(run.Name))
data, err := ioutil.ReadFile(config.DataDir("pws-slow-tasks", run.Name+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
allow := map[string]bool{
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.depot-client.run-container.creating-container-in-garden": true,
"rep.depot-client.delete-container.destroy.started": true,
"rep.depot-client.run-container.run.action.download-step.fetch-starting": true,
"rep.depot-client.run-container.run.monitor-run.run-step.running": true,
"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
"rep.depot-client.run-container.run.setup.download-step.fetch-starting": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
options := analyzers.SignificantEventsOptions{
LineOverlays: containerCountOverlays(entries),
}
if !run.CliffTimestamp.IsZero() {
options.MaxT = run.EndTimestamp.Add(time.Minute * 30)
options.VerticalMarkers = []analyzers.VerticalMarker{
{T: run.CliffTimestamp, LineStyle: viz.LineStyle(viz.Red, 1, viz.Dash)},
{T: run.EndTimestamp, LineStyle: viz.LineStyle(viz.Black, 1, viz.Dash)},
}
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("pws-slow-tasks", run.Name+".png"),
options,
)
}
}
func analyzeCPUWeightStresstest() {
runs := []string{
"unmodified-run",
"aufs-run",
"2-conc-run",
}
for _, run := range runs {
say.Println(0, say.Green(run))
data, err := ioutil.ReadFile(config.DataDir("cpu-wait-stress-test", run+".unified"))
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEvents(entries)
allow := map[string]bool{
"rep.auction-fetch-state.handling": true,
"rep.container-metrics-reporter.tick.started": true,
"rep.depot-client.run-container.creating-container-in-garden": true,
"rep.depot-client.delete-container.destroy.started": true,
"rep.depot-client.run-container.run.action.download-step.fetch-starting": true,
"rep.depot-client.run-container.run.monitor-run.run-step.running": true,
"rep.depot-client.run-container.run.run-step-process.step-finished-with-error": true,
"rep.depot-client.run-container.run.setup.download-step.fetch-starting": true,
}
filteredSignificantEvents := analyzers.SignificantEvents{}
for name, events := range significantEvents {
if allow[name] {
filteredSignificantEvents[name] = events
}
}
filteredSignificantEvents.LogWithThreshold(0.2)
options := analyzers.SignificantEventsOptions{
LineOverlays: cpuWeightStressTestContainerCountOverlays(entries),
}
analyzers.VisualizeSignificantEvents(
filteredSignificantEvents,
config.DataDir("cpu-wait-stress-test", run+".png"),
options,
)
}
}
func printTask(task *models.Task) {
privileged := ""
if task.Privileged {
privileged = say.Red(" PRIVILEGED")
}
say.Println(1,
"%s [%s on %s %s%s] U:%s C:%s (%d MB, %d MB, %d CPU)",
taskState(task),
task.TaskGuid,
task.CellId,
say.Cyan(task.RootFs),
privileged,
time.Since(time.Unix(0, task.UpdatedAt)).String(),
time.Since(time.Unix(0, task.CreatedAt)).String(),
task.MemoryMb,
task.DiskMb,
task.CpuWeight,
)
}
func analyzeEventDurations(path string, options analyzers.SignificantEventsOptions, n int, skips []string, outFile string) {
data, err := ioutil.ReadFile(path)
say.ExitIfError("couldn't read log file", err)
entries := util.ChugLagerEntries(data)
significantEvents := analyzers.ExtractSignificantEventsWithThreshold(entries, n)
significantEvents.LogWithThreshold(0.2)
for _, skip := range skips {
say.Println(0, "Skipping %s", skip)
delete(significantEvents, skip)
}
analyzers.VisualizeSignificantEvents(
significantEvents,
outFile,
options,
)
}
func DumpStoreCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
tasks bool
lrps bool
rate time.Duration
verbose bool
)
flagSet := flag.NewFlagSet("dump-store", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
flagSet.BoolVar(&tasks, "tasks", true, "print tasks")
flagSet.BoolVar(&lrps, "lrps", true, "print lrps")
flagSet.BoolVar(&verbose, "v", false, "be verbose")
flagSet.DurationVar(&rate, "rate", time.Duration(0), "rate at which to poll the store")
return common.Command{
Name: "dump-store",
Description: "- Fetch and print contents of the BBS",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
if rate == 0 {
err = dump(bbsClient, verbose, tasks, lrps, false)
common.ExitIfError("Failed to dump", err)
return
}
ticker := time.NewTicker(rate)
for {
<-ticker.C
err = dump(bbsClient, verbose, tasks, lrps, true)
if err != nil {
say.Println(0, say.Red("Failed to dump: %s", err.Error()))
}
}
},
}
}
func Vitals(vitalsAddrs map[string]string, out io.Writer) error {
client := &http.Client{
Transport: &http.Transport{},
Timeout: time.Second,
}
components := []string{}
for component := range vitalsAddrs {
components = append(components, component)
}
sort.Strings(components)
say.Println(0, "Vitals on %s", time.Now())
for _, component := range components {
dumpVitals(client, component, vitalsAddrs[component], out)
}
return nil
}
func (l *CliLogger) Println(i int, f string, a ...interface{}) {
say.Println(i, f, a...)
}
func GardenContainers(gardenAddr string, gardenNetwork string, raw bool, out io.Writer) error {
client := client.New(connection.New(gardenNetwork, gardenAddr))
containers, err := client.Containers(nil)
if err != nil {
return err
}
workPool, err := workpool.NewWorkPool(32)
if err != nil {
return err
}
lock := &sync.Mutex{}
wg := &sync.WaitGroup{}
wg.Add(len(containers))
containerInfos := []ContainerInfo{}
for _, container := range containers {
container := container
workPool.Submit(func() {
defer wg.Done()
info, err := container.Info()
if err != nil {
say.Println(1, say.Red("Failed to fetch container info: %s\n", container.Handle()))
return
}
metrics, err := container.Metrics()
if err != nil {
say.Println(1, say.Red("Failed to fetch container metrics: %s\n", container.Handle()))
return
}
lock.Lock()
defer lock.Unlock()
containerInfos = append(containerInfos, ContainerInfo{
container.Handle(),
info,
metrics,
})
})
}
wg.Wait()
if raw {
encoded, err := json.MarshalIndent(containerInfos, "", " ")
if err != nil {
return err
}
out.Write(encoded)
return nil
}
if len(containerInfos) == 0 {
say.Println(0, say.Red("No Containers"))
}
for _, containerInfo := range containerInfos {
printContainer(out, containerInfo)
}
return nil
}
func SetDomain(bbsClient bbs.Client, domain string, ttl time.Duration) error {
say.Println(0, say.Green("Setting Domain %s with TTL %ds", domain, int(ttl.Seconds())))
return bbsClient.UpsertDomain(domain, ttl)
}