// getActualResources returns the actual resource usage of the allocations.
func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) {
// Compute the total
total := computeNodeTotalResources(node)
// Get Resources
var cpu float64
var mem uint64
for _, alloc := range runningAllocs {
// Make the call to the client to get the actual usage.
stats, err := client.Allocations().Stats(alloc, nil)
if err != nil {
return nil, err
}
cpu += stats.ResourceUsage.CpuStats.TotalTicks
mem += stats.ResourceUsage.MemoryStats.RSS
}
resources := make([]string, 2)
resources[0] = "CPU|Memory"
resources[1] = fmt.Sprintf("%v/%v MHz|%v/%v",
math.Floor(cpu),
total.CPU,
humanize.IBytes(mem),
humanize.IBytes(uint64(total.MemoryMB*bytesPerMegabyte)))
return resources, nil
}
// followFile outputs the contents of the file to stdout relative to the end of
// the file.
func (l *LogsCommand) followFile(client *api.Client, alloc *api.Allocation,
follow bool, task, logType, origin string, offset int64) (io.ReadCloser, error) {
cancel := make(chan struct{})
frames, err := client.AllocFS().Logs(alloc, follow, task, logType, origin, offset, cancel, nil)
if err != nil {
return nil, err
}
signalCh := make(chan os.Signal, 1)
signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
// Create a reader
var r io.ReadCloser
frameReader := api.NewFrameReader(frames, cancel)
frameReader.SetUnblockTime(500 * time.Millisecond)
r = frameReader
go func() {
<-signalCh
// End the streaming
r.Close()
}()
return r, nil
}
// outputPeriodicInfo prints information about the passed periodic job. If a
// request fails, an error is returned.
func (c *StatusCommand) outputPeriodicInfo(client *api.Client, job *api.Job) error {
// Generate the prefix that matches launched jobs from the periodic job.
prefix := fmt.Sprintf("%s%s", job.ID, structs.PeriodicLaunchSuffix)
children, _, err := client.Jobs().PrefixList(prefix)
if err != nil {
return fmt.Errorf("Error querying job: %s", err)
}
if len(children) == 0 {
c.Ui.Output("\nNo instances of periodic job found")
return nil
}
out := make([]string, 1)
out[0] = "ID|Status"
for _, child := range children {
// Ensure that we are only showing jobs whose parent is the requested
// job.
if child.ParentID != job.ID {
continue
}
out = append(out, fmt.Sprintf("%s|%s",
child.ID,
child.Status))
}
c.Ui.Output(fmt.Sprintf("\nPreviously launched jobs:\n%s", formatList(out)))
return nil
}
// followFile outputs the contents of the file to stdout relative to the end of
// the file. If numLines does not equal -1, then tail -n behavior is used.
func (f *FSCommand) followFile(client *api.Client, alloc *api.Allocation,
path, origin string, offset, numLines int64) (io.ReadCloser, error) {
cancel := make(chan struct{})
frames, err := client.AllocFS().Stream(alloc, path, origin, offset, cancel, nil)
if err != nil {
return nil, err
}
signalCh := make(chan os.Signal, 1)
signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
// Create a reader
var r io.ReadCloser
frameReader := api.NewFrameReader(frames, cancel)
frameReader.SetUnblockTime(500 * time.Millisecond)
r = frameReader
// If numLines is set, wrap the reader
if numLines != -1 {
r = NewLineLimitReader(r, int(numLines), int(numLines*bytesToLines), 1*time.Second)
}
go func() {
<-signalCh
// End the streaming
r.Close()
}()
return r, nil
}
// regionLeaders returns a map of regions to the IP of the member that is the
// leader.
func regionLeaders(client *api.Client, mem []*api.AgentMember) (map[string]string, error) {
// Determine the unique regions.
leaders := make(map[string]string)
regions := make(map[string]struct{})
for _, m := range mem {
regions[m.Tags["region"]] = struct{}{}
}
if len(regions) == 0 {
return leaders, nil
}
status := client.Status()
for reg := range regions {
l, err := status.RegionLeader(reg)
if err != nil {
// This error means that region has no leader.
if strings.Contains(err.Error(), "No cluster leader") {
continue
}
return nil, err
}
leaders[reg] = l
}
return leaders, nil
}
// getRunningAllocs returns a slice of allocation id's running on the node
func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) {
var allocs []*api.Allocation
// Query the node allocations
nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil)
// Filter list to only running allocations
for _, alloc := range nodeAllocs {
if alloc.ClientStatus == "running" {
allocs = append(allocs, alloc)
}
}
return allocs, err
}
// outputJobInfo prints information about the passed non-periodic job. If a
// request fails, an error is returned.
func (c *StatusCommand) outputJobInfo(client *api.Client, job *api.Job) error {
var evals, allocs []string
// Query the evaluations
jobEvals, _, err := client.Jobs().Evaluations(job.ID, nil)
if err != nil {
return fmt.Errorf("Error querying job evaluations: %s", err)
}
// Query the allocations
jobAllocs, _, err := client.Jobs().Allocations(job.ID, nil)
if err != nil {
return fmt.Errorf("Error querying job allocations: %s", err)
}
// Format the evals
evals = make([]string, len(jobEvals)+1)
evals[0] = "ID|Priority|TriggeredBy|Status"
for i, eval := range jobEvals {
evals[i+1] = fmt.Sprintf("%s|%d|%s|%s",
eval.ID[:c.length],
eval.Priority,
eval.TriggeredBy,
eval.Status)
}
// Format the allocs
allocs = make([]string, len(jobAllocs)+1)
allocs[0] = "ID|EvalID|NodeID|TaskGroup|Desired|Status"
for i, alloc := range jobAllocs {
allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s",
alloc.ID[:c.length],
alloc.EvalID[:c.length],
alloc.NodeID[:c.length],
alloc.TaskGroup,
alloc.DesiredStatus,
alloc.ClientStatus)
}
c.Ui.Output("\n==> Evaluations")
c.Ui.Output(formatList(evals))
c.Ui.Output("\n==> Allocations")
c.Ui.Output(formatList(allocs))
return nil
}
// getAllocs returns information about every running allocation on the node
func getAllocs(client *api.Client, node *api.Node, length int) ([]string, error) {
var allocs []string
// Query the node allocations
nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil)
// Format the allocations
allocs = make([]string, len(nodeAllocs)+1)
allocs[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status"
for i, alloc := range nodeAllocs {
allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s",
limit(alloc.ID, length),
limit(alloc.EvalID, length),
alloc.JobID,
alloc.TaskGroup,
alloc.DesiredStatus,
alloc.ClientStatus)
}
return allocs, err
}
// getLocalNodeID returns the node ID of the local Nomad Client and an error if
// it couldn't be determined or the Agent is not running in Client mode.
func getLocalNodeID(client *api.Client) (string, error) {
info, err := client.Agent().Self()
if err != nil {
return "", fmt.Errorf("Error querying agent info: %s", err)
}
var stats map[string]interface{}
stats, _ = info["stats"]
clientStats, ok := stats["client"].(map[string]interface{})
if !ok {
return "", fmt.Errorf("Nomad not running in client mode")
}
nodeID, ok := clientStats["node_id"].(string)
if !ok {
return "", fmt.Errorf("Failed to determine node ID")
}
return nodeID, nil
}
// Get Random Allocation ID from a known jobID. Prefer to use a running allocation,
// but use a dead allocation if no running allocations are found
func getRandomJobAlloc(client *api.Client, jobID string) (string, error) {
var runningAllocs []*api.AllocationListStub
allocs, _, err := client.Jobs().Allocations(jobID, nil)
// Check that the job actually has allocations
if len(allocs) == 0 {
return "", fmt.Errorf("job %q doesn't exist or it has no allocations", jobID)
}
for _, v := range allocs {
if v.ClientStatus == "running" {
runningAllocs = append(runningAllocs, v)
}
}
// If we don't have any allocations running, use dead allocations
if len(runningAllocs) < 1 {
runningAllocs = allocs
}
r := rand.New(rand.NewSource(time.Now().UnixNano()))
allocID := runningAllocs[r.Intn(len(runningAllocs))].ID
return allocID, err
}
func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int {
// Format the header output
basic := []string{
fmt.Sprintf("ID|%s", limit(node.ID, c.length)),
fmt.Sprintf("Name|%s", node.Name),
fmt.Sprintf("Class|%s", node.NodeClass),
fmt.Sprintf("DC|%s", node.Datacenter),
fmt.Sprintf("Drain|%v", node.Drain),
fmt.Sprintf("Status|%s", node.Status),
}
if c.short {
c.Ui.Output(c.Colorize().Color(formatKV(basic)))
} else {
// Get the host stats
hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil)
if nodeStatsErr != nil {
c.Ui.Output("")
c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", nodeStatsErr))
}
if hostStats != nil {
uptime := time.Duration(hostStats.Uptime * uint64(time.Second))
basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String()))
}
c.Ui.Output(c.Colorize().Color(formatKV(basic)))
// Get list of running allocations on the node
runningAllocs, err := getRunningAllocs(client, node.ID)
if err != nil {
c.Ui.Error(fmt.Sprintf("Error querying node for running allocations: %s", err))
return 1
}
allocatedResources := getAllocatedResources(client, runningAllocs, node)
c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]"))
c.Ui.Output(formatList(allocatedResources))
actualResources, err := getActualResources(client, runningAllocs, node)
if err == nil {
c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]"))
c.Ui.Output(formatList(actualResources))
}
hostResources, err := getHostResources(hostStats, node)
if err != nil {
c.Ui.Output("")
c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", err))
}
if err == nil {
c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]"))
c.Ui.Output(formatList(hostResources))
}
if hostStats != nil && c.stats {
c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]"))
c.printCpuStats(hostStats)
c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]"))
c.printMemoryStats(hostStats)
c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]"))
c.printDiskStats(hostStats)
}
}
allocs, err := getAllocs(client, node, c.length)
if err != nil {
c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err))
return 1
}
if len(allocs) > 1 {
c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]"))
c.Ui.Output(formatList(allocs))
}
if c.verbose {
c.formatAttributes(node)
}
return 0
}
// outputJobInfo prints information about the passed non-periodic job. If a
// request fails, an error is returned.
func (c *StatusCommand) outputJobInfo(client *api.Client, job *api.Job) error {
var evals, allocs []string
// Query the allocations
jobAllocs, _, err := client.Jobs().Allocations(job.ID, nil)
if err != nil {
return fmt.Errorf("Error querying job allocations: %s", err)
}
// Query the evaluations
jobEvals, _, err := client.Jobs().Evaluations(job.ID, nil)
if err != nil {
return fmt.Errorf("Error querying job evaluations: %s", err)
}
// Determine latest evaluation with failures whose follow up hasn't
// completed, this is done while formatting
var latestFailedPlacement *api.Evaluation
blockedEval := false
// Format the evals
evals = make([]string, len(jobEvals)+1)
evals[0] = "ID|Priority|Triggered By|Status|Placement Failures"
for i, eval := range jobEvals {
failures, _ := evalFailureStatus(eval)
evals[i+1] = fmt.Sprintf("%s|%d|%s|%s|%s",
limit(eval.ID, c.length),
eval.Priority,
eval.TriggeredBy,
eval.Status,
failures,
)
if eval.Status == "blocked" {
blockedEval = true
}
if len(eval.FailedTGAllocs) == 0 {
// Skip evals without failures
continue
}
if latestFailedPlacement == nil || latestFailedPlacement.CreateIndex < eval.CreateIndex {
latestFailedPlacement = eval
}
}
if c.verbose || c.showEvals {
c.Ui.Output(c.Colorize().Color("\n[bold]Evaluations[reset]"))
c.Ui.Output(formatList(evals))
}
if blockedEval && latestFailedPlacement != nil {
c.outputFailedPlacements(latestFailedPlacement)
}
// Format the allocs
c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]"))
if len(jobAllocs) > 0 {
allocs = make([]string, len(jobAllocs)+1)
allocs[0] = "ID|Eval ID|Node ID|Task Group|Desired|Status"
for i, alloc := range jobAllocs {
allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s",
limit(alloc.ID, c.length),
limit(alloc.EvalID, c.length),
limit(alloc.NodeID, c.length),
alloc.TaskGroup,
alloc.DesiredStatus,
alloc.ClientStatus)
}
c.Ui.Output(formatList(allocs))
} else {
c.Ui.Output("No allocations placed")
}
return nil
}