func dumpVitals(client *http.Client, component string, addr string, out io.Writer) {
response, err := client.Get("http://" + addr + "/debug/pprof/")
if err != nil {
say.Println(0, say.Red("%s: %s"), component, err.Error())
return
}
if response.StatusCode != http.StatusOK {
say.Println(0, say.Red("%s: %d"), component, response.StatusCode)
return
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
say.Println(0, say.Red("%s: %d"), component, err.Error())
return
}
s := string(body)
report := []string{}
rows := strings.Split(s, "<tr>")[1:]
for _, row := range rows {
columns := strings.Split(row, "<td>")
value, _ := strconv.Atoi(strings.Split(columns[0], ">")[1])
name := strings.Split(columns[1], ">")[1]
name = name[:len(name)-3]
if value > 1000 {
report = append(report, say.Red("%20s", fmt.Sprintf("%d %s", value, name)))
} else {
report = append(report, fmt.Sprintf("%20s", fmt.Sprintf("%d %s", value, name)))
}
}
say.Println(0, "%s: %s %s", say.Green("%25s", component), string(strings.Join(report, " ")), addr)
}
func ExitIfError(context string, err error) {
if err != nil {
say.Fprintln(os.Stderr, 0, say.Red(context))
say.Fprintln(os.Stderr, 0, say.Red(err.Error()))
os.Exit(1)
}
}
func (s *stenographer) PrettyPrintLog(log chug.LogEntry) {
components := []string{}
color, ok := colorLookup[strings.Split(log.Source, ":")[0]]
if !ok {
color = say.DefaultStyle
}
level := ""
switch log.LogLevel {
case lager.INFO:
level = say.Colorize(color, "%-7s", "[INFO]")
case lager.DEBUG:
level = say.Gray("%-7s", "[DEBUG]")
case lager.ERROR:
level = say.Red("%-7s", "[ERROR]")
case lager.FATAL:
level = say.Red("%-7s", "[FATAL]")
}
var timestamp string
if s.Absolute {
timestamp = log.Timestamp.Format("01/02 15:04:05.00")
} else {
timestamp = log.Timestamp.Sub(s.RelativeTime).String()
timestamp = fmt.Sprintf("%17s", timestamp)
}
components = append(components, say.Colorize(color, "%-16s", log.Source))
components = append(components, level)
components = append(components, say.Colorize(color, timestamp))
components = append(components, say.Gray("%-10s", log.Session))
components = append(components, say.Colorize(color, log.Message))
say.Println(0, strings.Join(components, " "))
if log.Error != nil {
say.Println(27, say.Red("Error: %s", log.Error.Error()))
}
if log.Trace != "" {
say.Println(27, say.Red(log.Trace))
}
if len(log.Data) > 0 && s.Data == "short" {
dataJSON, _ := json.Marshal(log.Data)
say.Println(27, string(dataJSON))
}
if len(log.Data) > 0 && s.Data == "long" {
dataJSON, _ := json.MarshalIndent(log.Data, "", " ")
say.Println(27, string(dataJSON))
}
}
func usage(commandGroups []common.CommandGroup) {
if len(os.Args) > 2 {
matcher := strings.ToLower(os.Args[2])
for _, commandGroup := range commandGroups {
if strings.HasPrefix(strings.ToLower(commandGroup.Name), matcher) {
usageForCommandGroup(commandGroup, true)
return
}
for _, command := range commandGroup.Commands {
if strings.HasPrefix(strings.ToLower(command.Name), matcher) {
usageForCommand(0, command, true)
return
}
}
}
say.Fprintln(os.Stderr, 0, say.Red("Unkown command: %s", os.Args[2]))
}
say.Fprintln(os.Stderr, 0, "%s", say.Cyan("Help and Autocompletion"))
say.Fprintln(os.Stderr, 0, strings.Repeat("-", len("Help and Autocompletion")))
say.Fprintln(os.Stderr, 1, "%s %s", say.Green("help"), say.LightGray("[command] - Show this help, or detailed help for the passed in command"))
say.Fprintln(os.Stderr, 1, "%s %s", say.Green("completions"), say.LightGray("Generate BASH Completions for veritas"))
say.Fprintln(os.Stderr, 0, "")
for _, commandGroup := range commandGroups {
usageForCommandGroup(commandGroup, false)
say.Println(0, "")
}
}
func GetDesiredLRPCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("get-desired-lrp", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "get-desired-lrp",
Description: "<process-guid> - get a DesiredLRP",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
if len(args) == 0 {
say.Fprintln(os.Stderr, 0, say.Red("missing process-guid"))
os.Exit(1)
}
desiredLRP, err := bbsClient.DesiredLRPByProcessGuid(args[0])
common.ExitIfError("Failed to fetch DesiredLRP", err)
preview, _ := json.MarshalIndent(desiredLRP, "", " ")
say.Println(0, string(preview))
},
}
}
func SetDomainCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("set-domain", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "set-domain",
Description: "domain ttl - updates the domain in the BBS (ttl is a duration)",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
if len(args) != 2 {
say.Fprintln(os.Stderr, 0, say.Red("Expected domain and ttl"))
os.Exit(1)
}
ttl, err := time.ParseDuration(args[1])
common.ExitIfError("Failed to parse TTL", err)
err = set_domain.SetDomain(bbsClient, args[0], ttl)
common.ExitIfError("Failed to submit lrp", err)
},
}
}
func RemoveLRPCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("remove-lrp", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "remove-lrp",
Description: "<process-guid> - remove an lrp",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
if len(args) == 0 {
say.Fprintln(os.Stderr, 0, say.Red("You must specify a process-guid"))
os.Exit(1)
} else {
err := bbsClient.RemoveDesiredLRP(args[0])
common.ExitIfError("Failed to remove lrp", err)
}
},
}
}
func colorByTaskSuccess(task *models.Task, format string, args ...interface{}) string {
if task.Failed {
return say.Red(format, args...)
} else {
return say.Green(format, args...)
}
}
func UnifyChugCommand() common.Command {
var (
minTimeFlag string
maxTimeFlag string
matchFlag string
excludeFlag string
)
flagSet := baseFlagSet("chug-unify", &minTimeFlag, &maxTimeFlag, &matchFlag, &excludeFlag)
return common.Command{
Name: "chug-unify",
Description: "file1, file2,... - Combine lager files in temporal order",
FlagSet: flagSet,
Run: func(args []string) {
minTime, maxTime, match, exclude := parseBaseFlags(minTimeFlag, maxTimeFlag, matchFlag, excludeFlag)
if len(args) == 0 {
say.Println(0, say.Red("You must pass chug-unify files to combine"))
os.Exit(1)
} else {
files := []io.Reader{}
for _, arg := range args {
f, err := os.Open(arg)
common.ExitIfError("Could not open file", err)
files = append(files, f)
}
err := Unify(files, os.Stdout, minTime, maxTime, match, exclude)
common.ExitIfError("Failed to chug-unify", err)
}
},
}
}
func RepState(out io.Writer) (err error) {
client := rep.NewClient(&http.Client{
Timeout: 5 * time.Second,
}, "http://localhost:1800")
t := time.Now()
state, err := client.State()
dt := time.Since(t)
if err != nil {
say.Println(0, "Cell State [%s] - Error:%s", dt, say.Red(err.Error()))
return err
}
name := say.Green("Cell State")
if state.Evacuating {
name = say.Red("Cell State - EVAC -")
}
rootFSes := []string{}
for key := range state.RootFSProviders {
if key != "preloaded" {
rootFSes = append(rootFSes, say.Yellow(key))
}
}
for key := range state.RootFSProviders["preloaded"].(rep.FixedSetRootFSProvider).FixedSet {
rootFSes = append(rootFSes, say.Green("preloaded:%s", key))
}
say.Println(0, "%s [%s] - Zone:%s | %s Tasks, %s LRPs | C:%d/%d M:%d/%d D:%d/%d | %s",
name,
dt,
say.Cyan(state.Zone),
say.Cyan("%d", len(state.Tasks)),
say.Cyan("%d", len(state.LRPs)),
state.AvailableResources.Containers,
state.TotalResources.Containers,
state.AvailableResources.MemoryMB,
state.TotalResources.MemoryMB,
state.AvailableResources.DiskMB,
state.TotalResources.DiskMB,
strings.Join(rootFSes, ", "),
)
return nil
}
func actualState(actual *models.ActualLRP) string {
switch actual.State {
case models.ActualLRPStateUnclaimed:
if actual.PlacementError == "" {
return say.LightGray("UNCLAIMED")
} else {
return say.Red("UNCLAIMED (%s)", actual.PlacementError)
}
case models.ActualLRPStateClaimed:
return say.Yellow("CLAIMED")
case models.ActualLRPStateRunning:
return say.Green("RUNNING")
case models.ActualLRPStateCrashed:
return say.Red("CRASHED (%d - %s)", actual.CrashCount, strings.Replace(actual.CrashReason, "\n", " ", -1))
default:
return say.Red("INVALID")
}
}
func (s SignificantEvents) LogWithThreshold(threshold float64) {
for _, message := range s.OrderedNames() {
events := s[message]
s := fmt.Sprintf("%s %s", events.Data().Stats(), message)
if events.Data().Max() > threshold {
s = say.Red(s)
}
say.Println(0, s)
}
}
func monitor(route string, pollInterval, batchInterval time.Duration) {
say.Println(0, "Monitoring %s every %s", say.Green(route), say.Green("%s", pollInterval))
// http.DefaultClient.Timeout = 200 * time.Millisecond
ticker := time.NewTicker(pollInterval)
startTime := time.Now()
roundTime := time.Now()
indices := map[int]int{}
requests := 0
succesfulRequests := 0
for {
<-ticker.C
requests++
resp, err := http.Get(route + "/index")
if err != nil {
say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Error: %s", err.Error())))
continue
}
if resp.StatusCode != http.StatusOK {
say.Println(0, "%s: %s", say.Yellow("%s", time.Since(startTime)), say.Red(fmt.Sprintf("Invalid Status Code: %d", resp.StatusCode)))
say.Println(1, say.Red(format.Object(resp.Header, 0)))
continue
}
succesfulRequests++
indexStr, _ := ioutil.ReadAll(resp.Body)
index, _ := strconv.Atoi(string(indexStr))
indices[index]++
resp.Body.Close()
if time.Since(roundTime) >= batchInterval {
say.Println(0, "%s: %d/%d %s", say.Yellow("%s", time.Since(startTime)), succesfulRequests, requests, sortedIndices(indices))
indices = map[int]int{}
requests = 0
succesfulRequests = 0
roundTime = time.Now()
}
}
}
func printDomains(domains []string) {
say.Println(0, say.Green("Domains"))
if len(domains) == 0 {
say.Println(1, say.Red("None"))
return
}
for _, domain := range domains {
say.Println(1, say.Green("%s", domain))
}
}
func taskState(task *models.Task) string {
switch task.State {
case models.Task_Pending:
return say.LightGray("PENDING ")
case models.Task_Running:
return say.Yellow("RUNNING ")
case models.Task_Completed:
return colorByTaskSuccess(task, "COMPLETED")
case models.Task_Resolving:
return colorByTaskSuccess(task, "RESOLVING")
default:
return say.Red("INVALID")
}
}
func GenerateEventDurationCommand() say.Command {
var minT, maxT float64
var skipList string
var blueEvent, redEvent string
var outFile string
var significantThreshold int
var fs = &flag.FlagSet{}
fs.Float64Var(&minT, "tmin", 0, "Min time")
fs.Float64Var(&maxT, "tmax", 0, "Max time")
fs.StringVar(&skipList, "skip", "", "Events to skip (comma delimited)")
fs.StringVar(&blueEvent, "blue", "", "Events to use to generate blue markers")
fs.StringVar(&redEvent, "red", "", "Events to use to generate red markers")
fs.IntVar(&significantThreshold, "n", 2, "Minimum number of events required to make it onto the plot")
fs.StringVar(&outFile, "o", "", "Output file")
return say.Command{
Name: "event-duration",
Description: "lager.log -- generate event duration plots",
FlagSet: fs,
Run: func(args []string) {
if len(args) != 1 {
say.Println(0, say.Red("please provide a lager file to read"))
os.Exit(1)
}
markedEvents := map[string]plot.LineStyle{}
if blueEvent != "" {
markedEvents[blueEvent] = viz.LineStyle(viz.Blue, 1, viz.Dot)
}
if redEvent != "" {
markedEvents[redEvent] = viz.LineStyle(viz.Red, 1, viz.Dot)
}
options := analyzers.SignificantEventsOptions{
MinX: minT,
MaxX: maxT,
MarkedEvents: markedEvents,
}
skips := strings.Split(skipList, ",")
if outFile == "" {
outFile = "out.png"
}
analyzeEventDurations(args[0], options, significantThreshold, skips, outFile)
},
}
}
func GetActualLRPCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("get-actual-lrp", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "get-actual-lrp",
Description: "<process-guid> <optional: index> - get an ActualLRP",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
var index = -1
if len(args) == 0 {
say.Fprintln(os.Stderr, 0, say.Red("missing process-guid"))
os.Exit(1)
}
processGuid := args[0]
if len(args) == 2 {
index, err = strconv.Atoi(args[1])
common.ExitIfError("Could not parse index", err)
}
if index == -1 {
actualLRPGroups, err := bbsClient.ActualLRPGroupsByProcessGuid(processGuid)
common.ExitIfError("Could not fetch ActualLRPs", err)
for _, actualLRPGroup := range actualLRPGroups {
actualLRP, _ := actualLRPGroup.Resolve()
preview, _ := json.MarshalIndent(actualLRP, "", " ")
say.Println(0, string(preview))
}
} else {
actualLRPGroup, err := bbsClient.ActualLRPGroupByProcessGuidAndIndex(processGuid, index)
common.ExitIfError("Could not fetch ActualLRP", err)
actualLRP, _ := actualLRPGroup.Resolve()
preview, _ := json.MarshalIndent(actualLRP, "", " ")
say.Println(0, string(preview))
}
},
}
}
func UpdateDesiredLRPCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
)
flagSet := flag.NewFlagSet("update-lrp", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
return common.Command{
Name: "update-lrp",
Description: "<process-guid> <path to json file> - update a DesiredLRP",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
var raw = []byte{}
if len(args) == 0 {
say.Fprintln(os.Stderr, 0, say.Red("missing process-guid"))
os.Exit(1)
} else if len(args) == 1 {
say.Fprintln(os.Stderr, 0, "Reading from stdin...")
raw, err = ioutil.ReadAll(os.Stdin)
common.ExitIfError("Failed to read from stdin", err)
} else {
raw, err = ioutil.ReadFile(args[1])
common.ExitIfError("Failed to read from file", err)
}
desiredLRPUpdate := &models.DesiredLRPUpdate{}
err = json.Unmarshal(raw, desiredLRPUpdate)
common.ExitIfError("Failed to decode DesiredLRPUpdate", err)
say.Println(0, "Updating %s:", args[0])
preview, _ := json.MarshalIndent(desiredLRPUpdate, "", " ")
say.Println(0, string(preview))
err = bbsClient.UpdateDesiredLRP(args[0], desiredLRPUpdate)
common.ExitIfError("Failed to update DesiredLRP", err)
},
}
}
func printTask(task *models.Task) {
privileged := ""
if task.Privileged {
privileged = say.Red(" PRIVILEGED")
}
say.Println(1,
"%s [%s on %s %s%s] U:%s C:%s (%d MB, %d MB, %d CPU)",
taskState(task),
task.TaskGuid,
task.CellId,
say.Cyan(task.RootFs),
privileged,
time.Since(time.Unix(0, task.UpdatedAt)).String(),
time.Since(time.Unix(0, task.CreatedAt)).String(),
task.MemoryMb,
task.DiskMb,
task.CpuWeight,
)
}
func DumpStoreCommand() common.Command {
var (
bbsConfig config_finder.BBSConfig
tasks bool
lrps bool
rate time.Duration
verbose bool
)
flagSet := flag.NewFlagSet("dump-store", flag.ExitOnError)
bbsConfig.PopulateFlags(flagSet)
flagSet.BoolVar(&tasks, "tasks", true, "print tasks")
flagSet.BoolVar(&lrps, "lrps", true, "print lrps")
flagSet.BoolVar(&verbose, "v", false, "be verbose")
flagSet.DurationVar(&rate, "rate", time.Duration(0), "rate at which to poll the store")
return common.Command{
Name: "dump-store",
Description: "- Fetch and print contents of the BBS",
FlagSet: flagSet,
Run: func(args []string) {
bbsClient, err := config_finder.NewBBS(bbsConfig)
common.ExitIfError("Could not construct BBS", err)
if rate == 0 {
err = dump(bbsClient, verbose, tasks, lrps, false)
common.ExitIfError("Failed to dump", err)
return
}
ticker := time.NewTicker(rate)
for {
<-ticker.C
err = dump(bbsClient, verbose, tasks, lrps, true)
if err != nil {
say.Println(0, say.Red("Failed to dump: %s", err.Error()))
}
}
},
}
}
func (l *CliLogger) Red(f string, a ...interface{}) string {
return say.Red(f, a...)
}
func GardenContainers(gardenAddr string, gardenNetwork string, raw bool, out io.Writer) error {
client := client.New(connection.New(gardenNetwork, gardenAddr))
containers, err := client.Containers(nil)
if err != nil {
return err
}
workPool, err := workpool.NewWorkPool(32)
if err != nil {
return err
}
lock := &sync.Mutex{}
wg := &sync.WaitGroup{}
wg.Add(len(containers))
containerInfos := []ContainerInfo{}
for _, container := range containers {
container := container
workPool.Submit(func() {
defer wg.Done()
info, err := container.Info()
if err != nil {
say.Println(1, say.Red("Failed to fetch container info: %s\n", container.Handle()))
return
}
metrics, err := container.Metrics()
if err != nil {
say.Println(1, say.Red("Failed to fetch container metrics: %s\n", container.Handle()))
return
}
lock.Lock()
defer lock.Unlock()
containerInfos = append(containerInfos, ContainerInfo{
container.Handle(),
info,
metrics,
})
})
}
wg.Wait()
if raw {
encoded, err := json.MarshalIndent(containerInfos, "", " ")
if err != nil {
return err
}
out.Write(encoded)
return nil
}
if len(containerInfos) == 0 {
say.Println(0, say.Red("No Containers"))
}
for _, containerInfo := range containerInfos {
printContainer(out, containerInfo)
}
return nil
}
func printLRP(lrp *veritas_models.VeritasLRP) {
say.Println(1, say.Green(lrp.ProcessGuid))
if lrp.DesiredLRP.GetProcessGuid() != "" {
privileged := ""
if lrp.DesiredLRP.Privileged {
privileged = say.Red(" PRIVILEGED")
}
routesString := routes(lrp.DesiredLRP.Routes)
if routesString != "" {
routesString = "\n" + say.Indent(1, routesString)
}
say.Println(
2,
"%s %s%s (%d MB, %d MB, %d CPU)%s",
say.Green("%d", lrp.DesiredLRP.Instances),
say.Cyan(lrp.DesiredLRP.RootFs),
privileged,
lrp.DesiredLRP.MemoryMb,
lrp.DesiredLRP.DiskMb,
lrp.DesiredLRP.CpuWeight,
routesString,
)
} else {
say.Println(2, say.Red("UNDESIRED"))
}
orderedActualIndices := lrp.OrderedActualLRPIndices()
for _, index := range orderedActualIndices {
actualLRPGroup := lrp.ActualLRPGroupsByIndex[index]
if instance := actualLRPGroup.Instance; instance != nil {
if instance.State == models.ActualLRPStateUnclaimed || instance.State == models.ActualLRPStateCrashed {
say.Println(
3,
"%2s: [%s for %s]",
index,
actualState(instance),
time.Since(time.Unix(0, instance.Since)),
)
} else {
say.Println(
3,
"%2s: %s %s [%s for %s]",
index,
instance.InstanceGuid,
say.Yellow(instance.CellId),
actualState(instance),
time.Since(time.Unix(0, instance.Since)),
)
}
}
if evacuating := actualLRPGroup.Evacuating; evacuating != nil {
say.Println(
3,
"%s: %s %s [%s for %s] - %s",
say.Red("%2s", index),
say.Red(evacuating.InstanceGuid),
say.Yellow(evacuating.CellId),
actualState(evacuating),
time.Since(time.Unix(0, evacuating.Since)),
say.Red("EVACUATING"),
)
}
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
commandGroups := []common.CommandGroup{
common.CommandGroup{
Name: "Setup",
Description: "Commands to set veritas up on a BOSH Job",
Commands: []common.Command{
config_finder.AutodetectCommand(),
},
},
common.CommandGroup{
Name: "BBS",
Description: "Commands to fetch from the BBS",
Commands: []common.Command{
store.DumpStoreCommand(),
store.FetchStoreCommand(),
store.PrintStoreCommand(),
store.DistributionCommand(),
},
},
common.CommandGroup{
Name: "Chug",
Description: "Commands to prettify lager logs",
Commands: []common.Command{
chug.ChugCommand(),
chug.ServeChugCommand(),
chug.UnifyChugCommand(),
},
},
common.CommandGroup{
Name: "Components",
Description: "Commands to fetch information from various components",
Commands: []common.Command{
components.RepStateCommand(),
components.GardenContainersCommand(),
components.VitalsCommand(),
},
},
common.CommandGroup{
Name: "LRPS " + say.Red("[DANGER]"),
Description: "Commands to remove DesiredLRPs and modify Domains",
Commands: []common.Command{
lrps.CreateDesiredLRPCommand(),
lrps.UpdateDesiredLRPCommand(),
lrps.RemoveLRPCommand(),
lrps.GetDesiredLRPCommand(),
lrps.GetActualLRPCommand(),
lrps.SetDomainCommand(),
},
},
}
if len(os.Args) == 1 || os.Args[1] == "help" {
usage(commandGroups)
os.Exit(1)
}
if os.Args[1] == "completions" {
completions(commandGroups)
os.Exit(0)
}
for _, commandGroup := range commandGroups {
for _, command := range commandGroup.Commands {
if command.Name == os.Args[1] {
command.FlagSet.Parse(os.Args[2:])
command.Run(command.FlagSet.Args())
os.Exit(0)
}
}
}
say.Println(0, say.Red("Unkown command: %s", os.Args[1]))
usage(commandGroups)
}