// FindMissingLivenessProbes inspects all PodSpecs for missing liveness probes and generates a list of non-duplicate markers
func FindMissingLivenessProbes(g osgraph.Graph, f osgraph.Namer, setProbeCommand string) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastPodSpecNode := range g.NodesByKind(kubegraph.PodSpecNodeKind) {
podSpecNode := uncastPodSpecNode.(*kubegraph.PodSpecNode)
if hasLivenessProbe(podSpecNode) {
continue
}
topLevelNode := osgraph.GetTopLevelContainerNode(g, podSpecNode)
// skip any podSpec nodes that are managed by other nodes.
// Liveness probes should only be applied to a controlling
// podSpec node, and not to any of its children.
if hasControllerRefEdge(g, topLevelNode) {
continue
}
topLevelString := f.ResourceName(topLevelNode)
markers = append(markers, osgraph.Marker{
Node: podSpecNode,
RelatedNodes: []graph.Node{topLevelNode},
Severity: osgraph.InfoSeverity,
Key: MissingLivenessProbeWarning,
Message: fmt.Sprintf("%s has no liveness probe to verify pods are still running.",
topLevelString),
Suggestion: osgraph.Suggestion(fmt.Sprintf("%s %s --liveness ...", setProbeCommand, topLevelString)),
})
}
return markers
}
// FindRestartingPods inspects all Pods to see if they've restarted more than the threshold
func FindRestartingPods(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastPodNode := range g.NodesByKind(kubegraph.PodNodeKind) {
podNode := uncastPodNode.(*kubegraph.PodNode)
pod, ok := podNode.Object().(*kapi.Pod)
if !ok {
continue
}
for _, containerStatus := range pod.Status.ContainerStatuses {
if containerStatus.RestartCount >= RestartThreshold {
markers = append(markers, osgraph.Marker{
Node: podNode,
Severity: osgraph.WarningSeverity,
Key: RestartingPodWarning,
Message: fmt.Sprintf("container %q in %s has restarted %d times", containerStatus.Name,
f.ResourceName(podNode), containerStatus.RestartCount),
})
}
}
}
return markers
}
// FindDeploymentConfigReadinessWarnings inspects deploymentconfigs and reports those that
// don't have readiness probes set up.
func FindDeploymentConfigReadinessWarnings(g osgraph.Graph, f osgraph.Namer, setProbeCommand string) []osgraph.Marker {
markers := []osgraph.Marker{}
Node:
for _, uncastDcNode := range g.NodesByKind(deploygraph.DeploymentConfigNodeKind) {
dcNode := uncastDcNode.(*deploygraph.DeploymentConfigNode)
if t := dcNode.DeploymentConfig.Spec.Template; t != nil && len(t.Spec.Containers) > 0 {
for _, container := range t.Spec.Containers {
if container.ReadinessProbe != nil {
continue Node
}
}
// All of the containers in the deployment config lack a readiness probe
markers = append(markers, osgraph.Marker{
Node: uncastDcNode,
Severity: osgraph.WarningSeverity,
Key: MissingReadinessProbeWarning,
Message: fmt.Sprintf("%s has no readiness probe to verify pods are ready to accept traffic or ensure deployment is successful.",
f.ResourceName(dcNode)),
Suggestion: osgraph.Suggestion(fmt.Sprintf("%s %s --readiness ...", setProbeCommand, f.ResourceName(dcNode))),
})
continue Node
}
}
return markers
}
// FindMissingSecrets inspects all PodSpecs for any Secret reference that is a synthetic node (not a pre-existing node in the graph)
func FindMissingSecrets(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastPodSpecNode := range g.NodesByKind(kubegraph.PodSpecNodeKind) {
podSpecNode := uncastPodSpecNode.(*kubegraph.PodSpecNode)
missingSecrets := CheckMissingMountedSecrets(g, podSpecNode)
topLevelNode := osgraph.GetTopLevelContainerNode(g, podSpecNode)
topLevelString := f.ResourceName(topLevelNode)
for _, missingSecret := range missingSecrets {
markers = append(markers, osgraph.Marker{
Node: podSpecNode,
RelatedNodes: []graph.Node{missingSecret},
Severity: osgraph.WarningSeverity,
Key: UnmountableSecretWarning,
Message: fmt.Sprintf("%s is attempting to mount a missing secret %s",
topLevelString, f.ResourceName(missingSecret)),
})
}
}
return markers
}
// FindCircularBuilds checks all build configs for cycles
func FindCircularBuilds(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
// Filter out all but ImageStreamTag and BuildConfig nodes
nodeFn := osgraph.NodesOfKind(imagegraph.ImageStreamTagNodeKind, buildgraph.BuildConfigNodeKind)
// Filter out all but BuildInputImage and BuildOutput edges
edgeFn := osgraph.EdgesOfKind(buildedges.BuildInputImageEdgeKind, buildedges.BuildOutputEdgeKind)
// Create desired subgraph
sub := g.Subgraph(nodeFn, edgeFn)
markers := []osgraph.Marker{}
// Check for cycles
for _, cycle := range topo.CyclesIn(sub) {
nodeNames := []string{}
for _, node := range cycle {
nodeNames = append(nodeNames, f.ResourceName(node))
}
markers = append(markers, osgraph.Marker{
Node: cycle[0],
RelatedNodes: cycle,
Severity: osgraph.WarningSeverity,
Key: CyclicBuildConfigWarning,
Message: fmt.Sprintf("Cycle detected in build configurations: %s", strings.Join(nodeNames, " -> ")),
})
}
return markers
}
// findPendingTagMarkers is the guts behind FindPendingTags .... break out some of the content and reduce some indentation
func findPendingTagMarkers(istNode *imagegraph.ImageStreamTagNode, g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
buildFound := false
bcNodes := buildedges.BuildConfigsForTag(g, graph.Node(istNode))
for _, bcNode := range bcNodes {
latestBuild := buildedges.GetLatestBuild(g, bcNode)
// A build config points to the non existent tag but no current build exists.
if latestBuild == nil {
continue
}
buildFound = true
// A build config points to the non existent tag but something is going on with
// the latest build.
// TODO: Handle other build phases.
switch latestBuild.Build.Status.Phase {
case buildapi.BuildPhaseCancelled:
// TODO: Add a warning here.
case buildapi.BuildPhaseError:
// TODO: Add a warning here.
case buildapi.BuildPhaseComplete:
// We should never hit this. The output of our build is missing but the build is complete.
// Most probably the user has messed up?
case buildapi.BuildPhaseFailed:
// Since the tag hasn't been populated yet, we assume there hasn't been a successful
// build so far.
markers = append(markers, osgraph.Marker{
Node: graph.Node(latestBuild),
RelatedNodes: []graph.Node{graph.Node(istNode), graph.Node(bcNode)},
Severity: osgraph.ErrorSeverity,
Key: LatestBuildFailedErr,
Message: fmt.Sprintf("%s has failed.", f.ResourceName(latestBuild)),
Suggestion: osgraph.Suggestion(fmt.Sprintf("Inspect the build failure with 'oc logs -f bc/%s'", bcNode.BuildConfig.GetName())),
})
default:
// Do nothing when latest build is new, pending, or running.
}
}
// if no current builds exist for any of the build configs, append marker for that
// but ignore ISTs which have no build configs
if !buildFound && len(bcNodes) > 0 {
markers = append(markers, osgraph.Marker{
Node: graph.Node(istNode),
RelatedNodes: bcNodesToRelatedNodes(bcNodes),
Severity: osgraph.WarningSeverity,
Key: TagNotAvailableWarning,
Message: fmt.Sprintf("%s needs to be imported or created by a build.", f.ResourceName(istNode)),
Suggestion: osgraph.Suggestion(multiBCStartBuildSuggestion(bcNodes)),
})
}
return markers
}
// getImageStreamTagSuggestion will return the appropriate marker Suggestion for when a BuildConfig is missing its input ImageStreamTag; in particular,
// it will determine whether or not another BuildConfig can produce the aforementioned ImageStreamTag
func getImageStreamTagSuggestion(g osgraph.Graph, f osgraph.Namer, tagNode *imagegraph.ImageStreamTagNode) osgraph.Suggestion {
bcs := []string{}
for _, bcNode := range g.PredecessorNodesByEdgeKind(tagNode, buildedges.BuildOutputEdgeKind) {
bcs = append(bcs, f.ResourceName(bcNode))
}
if len(bcs) == 1 {
return osgraph.Suggestion(fmt.Sprintf("oc start-build %s", bcs[0]))
}
if len(bcs) > 0 {
return osgraph.Suggestion(fmt.Sprintf("`oc start-build` with one of these: %s.", strings.Join(bcs[:], ",")))
}
return osgraph.Suggestion(fmt.Sprintf("%s needs to be imported.", f.ResourceName(tagNode)))
}
func createMissingScaleRefMarker(hpaNode *kubenodes.HorizontalPodAutoscalerNode, scaleRef graphapi.Node, namer osgraph.Namer) osgraph.Marker {
return osgraph.Marker{
Node: hpaNode,
Severity: osgraph.ErrorSeverity,
RelatedNodes: []graphapi.Node{scaleRef},
Key: HPAMissingScaleRefError,
Message: fmt.Sprintf("%s is attempting to scale %s/%s, which doesn't exist",
namer.ResourceName(hpaNode),
hpaNode.HorizontalPodAutoscaler.Spec.ScaleRef.Kind,
hpaNode.HorizontalPodAutoscaler.Spec.ScaleRef.Name,
),
}
}
// FindOverlappingHPAs scans the graph in search of HorizontalPodAutoscalers that are attempting to scale the same set of pods.
// This can occur in two ways:
// - 1. label selectors for two ReplicationControllers/DeploymentConfigs/etc overlap
// - 2. multiple HorizontalPodAutoscalers are attempting to scale the same ReplicationController/DeploymentConfig/etc
// Case 1 is handled by deconflicting the area of influence of ReplicationControllers/DeploymentConfigs/etc, and therefore we
// can assume that it will be handled before this step. Therefore, we are only concerned with finding HPAs that are trying to
// scale the same resources.
//
// The algorithm that is used to implement this check is described as follows:
// - create a sub-graph containing only HPA nodes and other nodes that can be scaled, as well as any scaling edges or other
// edges used to connect between objects that can be scaled
// - for every resulting edge in the new sub-graph, create an edge in the reverse direction
// - find the shortest paths between all HPA nodes in the graph
// - shortest paths connecting two horizontal pod autoscalers are used to create markers for the graph
func FindOverlappingHPAs(graph osgraph.Graph, namer osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
nodeFilter := osgraph.NodesOfKind(
kubenodes.HorizontalPodAutoscalerNodeKind,
kubenodes.ReplicationControllerNodeKind,
deploynodes.DeploymentConfigNodeKind,
)
edgeFilter := osgraph.EdgesOfKind(
kubegraph.ScalingEdgeKind,
deploygraph.DeploymentEdgeKind,
kubeedges.ManagedByControllerEdgeKind,
)
hpaSubGraph := graph.Subgraph(nodeFilter, edgeFilter)
for _, edge := range hpaSubGraph.Edges() {
osgraph.AddReversedEdge(hpaSubGraph, edge.From(), edge.To(), sets.NewString())
}
hpaNodes := hpaSubGraph.NodesByKind(kubenodes.HorizontalPodAutoscalerNodeKind)
for _, firstHPA := range hpaNodes {
// we can use Dijkstra's algorithm as we know we do not have any negative edge weights
shortestPaths := path.DijkstraFrom(firstHPA, hpaSubGraph)
for _, secondHPA := range hpaNodes {
if firstHPA == secondHPA {
continue
}
shortestPath, _ := shortestPaths.To(secondHPA)
if shortestPath == nil {
// if two HPAs have no path between them, no error exists
continue
}
markers = append(markers, osgraph.Marker{
Node: firstHPA,
Severity: osgraph.WarningSeverity,
RelatedNodes: shortestPath[1:],
Key: HPAOverlappingScaleRefWarning,
Message: fmt.Sprintf("%s and %s overlap because they both attempt to scale %s",
namer.ResourceName(firstHPA), namer.ResourceName(secondHPA), nameList(shortestPath[1:len(shortestPath)-1], namer)),
})
}
}
return markers
}
// humanReadableOutput traverses the provided graph using DFS and outputs it
// in a human-readable format. It starts from the provided root, assuming it
// is an imageStreamTag node and continues to the rest of the graph handling
// only imageStreamTag and buildConfig nodes.
func (d *ChainDescriber) humanReadableOutput(g osgraph.Graph, f osgraph.Namer, root graph.Node, reverse bool) string {
if reverse {
g = g.EdgeSubgraph(osgraph.ReverseExistingDirectEdge)
}
var singleNamespace bool
if len(d.namespaces) == 1 && !d.namespaces.Has(kapi.NamespaceAll) {
singleNamespace = true
}
depth := map[graph.Node]int{
root: 0,
}
out := ""
dfs := &DepthFirst{
Visit: func(u, v graph.Node) {
depth[v] = depth[u] + 1
},
}
until := func(node graph.Node) bool {
var info string
switch t := node.(type) {
case *imagegraph.ImageStreamTagNode:
info = outputHelper(f.ResourceName(t), t.Namespace, singleNamespace)
case *buildgraph.BuildConfigNode:
info = outputHelper(f.ResourceName(t), t.BuildConfig.Namespace, singleNamespace)
default:
panic("this graph contains node kinds other than imageStreamTags and buildConfigs")
}
if depth[node] != 0 {
out += "\n"
}
out += fmt.Sprintf("%s", strings.Repeat("\t", depth[node]))
out += fmt.Sprintf("%s", info)
return false
}
dfs.Walk(g, root, until)
return out
}
// nameList outputs a nicely-formatted list of names:
// - given nodes ['a', 'b', 'c'], this will return "one of a, b, or c"
// - given nodes ['a', 'b'], this will return "a or b"
// - given nodes ['a'], this will return "a"
func nameList(nodes []graphapi.Node, namer osgraph.Namer) string {
names := []string{}
for _, node := range nodes {
names = append(names, namer.ResourceName(node))
}
switch len(names) {
case 0:
return ""
case 1:
return names[0]
case 2:
return names[0] + " or " + names[1]
default:
return "one of " + strings.Join(names[:len(names)-1], ", ") + ", or " + names[len(names)-1]
}
}
// FindHPASpecsMissingCPUTargets scans the graph in search of HorizontalPodAutoscalers that are missing a CPU utilization target.
// As of right now, the only metric that HPAs can use to scale pods is the CPU utilization, so if a HPA is missing this target it
// is effectively useless.
func FindHPASpecsMissingCPUTargets(graph osgraph.Graph, namer osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastNode := range graph.NodesByKind(kubenodes.HorizontalPodAutoscalerNodeKind) {
node := uncastNode.(*kubenodes.HorizontalPodAutoscalerNode)
if node.HorizontalPodAutoscaler.Spec.CPUUtilization == nil {
markers = append(markers, osgraph.Marker{
Node: node,
Severity: osgraph.ErrorSeverity,
Key: HPAMissingCPUTargetError,
Message: fmt.Sprintf("%s is missing a CPU utilization target", namer.ResourceName(node)),
Suggestion: osgraph.Suggestion(fmt.Sprintf(`oc patch %s -p '{"spec":{"cpuUtilization":{"targetPercentage": 80}}}'`, namer.ResourceName(node))),
})
}
}
return markers
}
func pvcMarker(g osgraph.Graph, f osgraph.Namer, dcNode *deploygraph.DeploymentConfigNode) *osgraph.Marker {
for _, uncastPvcNode := range g.SuccessorNodesByEdgeKind(dcNode, deployedges.VolumeClaimEdgeKind) {
pvcNode := uncastPvcNode.(*kubegraph.PersistentVolumeClaimNode)
if !pvcNode.Found() {
return &osgraph.Marker{
Node: dcNode,
RelatedNodes: []graph.Node{uncastPvcNode},
Severity: osgraph.WarningSeverity,
Key: MissingPVCWarning,
Message: fmt.Sprintf("%s points to a missing persistent volume claim (%s).", f.ResourceName(dcNode), f.ResourceName(pvcNode)),
// TODO: Suggestion: osgraph.Suggestion(fmt.Sprintf("oc create pvc ...")),
}
}
dc := dcNode.DeploymentConfig
rollingParams := dc.Spec.Strategy.RollingParams
isBlockedBySize := dc.Spec.Replicas > 1
isBlockedRolling := rollingParams != nil && rollingParams.MaxSurge.IntValue() > 0
// If the claim is not RWO or deployments will not have more than a pod running at any time
// then they should be fine.
if !hasRWOAccess(pvcNode) || (!isBlockedRolling && !isBlockedBySize) {
continue
}
// This shouldn't be an issue on single-host clusters but they are not the common case anyway.
// If github.com/kubernetes/kubernetes/issues/26567 ever gets fixed upstream, then we can drop
// this warning.
return &osgraph.Marker{
Node: dcNode,
RelatedNodes: []graph.Node{uncastPvcNode},
Severity: osgraph.WarningSeverity,
Key: SingleHostVolumeWarning,
Message: fmt.Sprintf("%s references a volume which may only be used in a single pod at a time - this may lead to hung deployments", f.ResourceName(dcNode)),
}
}
return nil
}
// FindMissingPortMapping checks all routes and reports those that don't specify a port while
// the service they are routing to, has multiple ports. Also if a service for a route doesn't
// exist, will be reported.
func FindMissingPortMapping(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
route:
for _, uncastRouteNode := range g.NodesByKind(routegraph.RouteNodeKind) {
for _, uncastServiceNode := range g.SuccessorNodesByEdgeKind(uncastRouteNode, routeedges.ExposedThroughRouteEdgeKind) {
routeNode := uncastRouteNode.(*routegraph.RouteNode)
svcNode := uncastServiceNode.(*kubegraph.ServiceNode)
if !svcNode.Found() {
markers = append(markers, osgraph.Marker{
Node: routeNode,
RelatedNodes: []graph.Node{svcNode},
Severity: osgraph.WarningSeverity,
Key: MissingServiceWarning,
Message: fmt.Sprintf("%s is supposed to route traffic to %s but %s doesn't exist.",
f.ResourceName(routeNode), f.ResourceName(svcNode), f.ResourceName(svcNode)),
})
continue route
}
if len(svcNode.Spec.Ports) > 1 && (routeNode.Spec.Port == nil || len(routeNode.Spec.Port.TargetPort.String()) == 0) {
markers = append(markers, osgraph.Marker{
Node: routeNode,
RelatedNodes: []graph.Node{svcNode},
Severity: osgraph.WarningSeverity,
Key: MissingRoutePortWarning,
Message: fmt.Sprintf("%s doesn't have a port specified and is routing traffic to %s which uses multiple ports.",
f.ResourceName(routeNode), f.ResourceName(svcNode)),
})
continue route
}
}
}
return markers
}
// FindUnmountableSecrets inspects all PodSpecs for any Secret reference that isn't listed as mountable by the referenced ServiceAccount
func FindUnmountableSecrets(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastPodSpecNode := range g.NodesByKind(kubegraph.PodSpecNodeKind) {
podSpecNode := uncastPodSpecNode.(*kubegraph.PodSpecNode)
unmountableSecrets := CheckForUnmountableSecrets(g, podSpecNode)
topLevelNode := osgraph.GetTopLevelContainerNode(g, podSpecNode)
topLevelString := f.ResourceName(topLevelNode)
saString := "MISSING_SA"
saNodes := g.SuccessorNodesByEdgeKind(podSpecNode, kubeedges.ReferencedServiceAccountEdgeKind)
if len(saNodes) > 0 {
saString = f.ResourceName(saNodes[0])
}
for _, unmountableSecret := range unmountableSecrets {
markers = append(markers, osgraph.Marker{
Node: podSpecNode,
RelatedNodes: []graph.Node{unmountableSecret},
Severity: osgraph.WarningSeverity,
Key: UnmountableSecretWarning,
Message: fmt.Sprintf("%s is attempting to mount a secret %s disallowed by %s",
topLevelString, f.ResourceName(unmountableSecret), saString),
})
}
}
return markers
}
// FindRouteAdmissionFailures creates markers for any routes that were rejected by their routers
func FindRouteAdmissionFailures(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastRouteNode := range g.NodesByKind(routegraph.RouteNodeKind) {
routeNode := uncastRouteNode.(*routegraph.RouteNode)
Route:
for _, ingress := range routeNode.Status.Ingress {
switch status, condition := routeapi.IngressConditionStatus(&ingress, routeapi.RouteAdmitted); status {
case kapi.ConditionFalse:
markers = append(markers, osgraph.Marker{
Node: routeNode,
Severity: osgraph.ErrorSeverity,
Key: RouteNotAdmittedTypeErr,
Message: fmt.Sprintf("%s was not accepted by router %q: %s (%s)", f.ResourceName(routeNode), ingress.RouterName, condition.Message, condition.Reason),
})
break Route
}
}
}
return markers
}
// FindUnpushableBuildConfigs checks all build configs that will output to an IST backed by an ImageStream and checks to make sure their builds can push.
func FindUnpushableBuildConfigs(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
// note, unlike with Inputs, ImageStreamImage is not a valid type for build output
bc:
for _, bcNode := range g.NodesByKind(buildgraph.BuildConfigNodeKind) {
for _, istNode := range g.SuccessorNodesByEdgeKind(bcNode, buildedges.BuildOutputEdgeKind) {
for _, uncastImageStreamNode := range g.SuccessorNodesByEdgeKind(istNode, imageedges.ReferencedImageStreamGraphEdgeKind) {
imageStreamNode := uncastImageStreamNode.(*imagegraph.ImageStreamNode)
if !imageStreamNode.IsFound {
markers = append(markers, osgraph.Marker{
Node: bcNode,
RelatedNodes: []graph.Node{istNode},
Severity: osgraph.ErrorSeverity,
Key: MissingOutputImageStreamErr,
Message: fmt.Sprintf("%s is pushing to %s, but the image stream for that tag does not exist.",
f.ResourceName(bcNode), f.ResourceName(istNode)),
})
continue
}
if len(imageStreamNode.Status.DockerImageRepository) == 0 {
markers = append(markers, osgraph.Marker{
Node: bcNode,
RelatedNodes: []graph.Node{istNode},
Severity: osgraph.ErrorSeverity,
Key: MissingRequiredRegistryErr,
Message: fmt.Sprintf("%s is pushing to %s, but the administrator has not configured the integrated Docker registry.",
f.ResourceName(bcNode), f.ResourceName(istNode)),
Suggestion: osgraph.Suggestion("oc adm registry -h"),
})
continue bc
}
}
}
}
return markers
}
// FindPendingTags inspects all imageStreamTags that serve as outputs to builds.
//
// Precedence of failures:
// 1. A build config points to the non existent tag but no current build exists.
// 2. A build config points to the non existent tag but the latest build has failed.
func FindPendingTags(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastIstNode := range g.NodesByKind(imagegraph.ImageStreamTagNodeKind) {
istNode := uncastIstNode.(*imagegraph.ImageStreamTagNode)
if bcNode, points := buildPointsToTag(g, uncastIstNode); points && !istNode.Found() {
latestBuild := latestBuild(g, bcNode)
// A build config points to the non existent tag but no current build exists.
if latestBuild == nil {
markers = append(markers, osgraph.Marker{
Node: graph.Node(bcNode),
RelatedNodes: []graph.Node{uncastIstNode},
Severity: osgraph.WarningSeverity,
Key: TagNotAvailableWarning,
Message: fmt.Sprintf("%s needs to be imported or created by a build.", f.ResourceName(istNode)),
Suggestion: osgraph.Suggestion(fmt.Sprintf("oc start-build %s", f.ResourceName(bcNode))),
})
continue
}
// A build config points to the non existent tag but something is going on with
// the latest build.
// TODO: Handle other build phases.
switch latestBuild.Build.Status.Phase {
case buildapi.BuildPhaseCancelled:
// TODO: Add a warning here.
case buildapi.BuildPhaseError:
// TODO: Add a warning here.
case buildapi.BuildPhaseComplete:
// We should never hit this. The output of our build is missing but the build is complete.
// Most probably the user has messed up?
case buildapi.BuildPhaseFailed:
// Since the tag hasn't been populated yet, we assume there hasn't been a successful
// build so far.
markers = append(markers, osgraph.Marker{
Node: graph.Node(latestBuild),
RelatedNodes: []graph.Node{uncastIstNode, graph.Node(bcNode)},
Severity: osgraph.ErrorSeverity,
Key: LatestBuildFailedErr,
Message: fmt.Sprintf("%s has failed.", f.ResourceName(latestBuild)),
Suggestion: osgraph.Suggestion(fmt.Sprintf("Inspect the build failure with 'oc logs %s'", f.ResourceName(latestBuild))),
})
default:
// Do nothing when latest build is new, pending, or running.
}
}
}
return markers
}
// FindDeploymentConfigTriggerErrors checks for possible failures in deployment config
// image change triggers.
//
// Precedence of failures:
// 1. The image stream for the tag of interest does not exist.
// 2. The image stream tag does not exist.
func FindDeploymentConfigTriggerErrors(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
dc:
for _, uncastDcNode := range g.NodesByKind(deploygraph.DeploymentConfigNodeKind) {
for _, uncastIstNode := range g.PredecessorNodesByEdgeKind(uncastDcNode, deployedges.TriggersDeploymentEdgeKind) {
if istNode := uncastIstNode.(*imagegraph.ImageStreamTagNode); !istNode.Found() {
dcNode := uncastDcNode.(*deploygraph.DeploymentConfigNode)
// The image stream for the tag of interest does not exist.
// TODO: Suggest `oc create imagestream` once we have that.
if isNode, exists := doesImageStreamExist(g, uncastIstNode); !exists {
markers = append(markers, osgraph.Marker{
Node: uncastDcNode,
RelatedNodes: []graph.Node{uncastIstNode, isNode},
Severity: osgraph.ErrorSeverity,
Key: MissingImageStreamErr,
Message: fmt.Sprintf("The image trigger for %s will have no effect because %s does not exist.",
f.ResourceName(dcNode), f.ResourceName(isNode)),
})
continue dc
}
// The image stream tag of interest does not exist.
markers = append(markers, osgraph.Marker{
Node: uncastDcNode,
RelatedNodes: []graph.Node{uncastIstNode},
Severity: osgraph.WarningSeverity,
Key: MissingImageStreamTagWarning,
Message: fmt.Sprintf("The image trigger for %s will have no effect until %s is imported or created by a build.",
f.ResourceName(dcNode), f.ResourceName(istNode)),
})
continue dc
}
}
}
return markers
}
// ictMarker inspects the image change triggers for the provided deploymentconfig and returns
// a marker in case of the following two scenarios:
//
// 1. The image stream pointed by the dc trigger doen not exist.
// 2. The image stream tag pointed by the dc trigger does not exist and there is no build in
// flight that could push to the tag.
func ictMarker(g osgraph.Graph, f osgraph.Namer, dcNode *deploygraph.DeploymentConfigNode) *osgraph.Marker {
for _, uncastIstNode := range g.PredecessorNodesByEdgeKind(dcNode, deployedges.TriggersDeploymentEdgeKind) {
if istNode := uncastIstNode.(*imagegraph.ImageStreamTagNode); !istNode.Found() {
// The image stream for the tag of interest does not exist.
if isNode, exists := doesImageStreamExist(g, uncastIstNode); !exists {
return &osgraph.Marker{
Node: dcNode,
RelatedNodes: []graph.Node{uncastIstNode, isNode},
Severity: osgraph.ErrorSeverity,
Key: MissingImageStreamErr,
Message: fmt.Sprintf("The image trigger for %s will have no effect because %s does not exist.",
f.ResourceName(dcNode), f.ResourceName(isNode)),
// TODO: Suggest `oc create imagestream` once we have that.
}
}
for _, bcNode := range buildedges.BuildConfigsForTag(g, istNode) {
// Avoid warning for the dc image trigger in case there is a build in flight.
if latestBuild := buildedges.GetLatestBuild(g, bcNode); latestBuild != nil && !buildutil.IsBuildComplete(latestBuild.Build) {
return nil
}
}
// The image stream tag of interest does not exist.
return &osgraph.Marker{
Node: dcNode,
RelatedNodes: []graph.Node{uncastIstNode},
Severity: osgraph.WarningSeverity,
Key: MissingImageStreamTagWarning,
Message: fmt.Sprintf("The image trigger for %s will have no effect until %s is imported or created by a build.",
f.ResourceName(dcNode), f.ResourceName(istNode)),
}
}
}
return nil
}
func FindDuelingReplicationControllers(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastRCNode := range g.NodesByKind(kubegraph.ReplicationControllerNodeKind) {
rcNode := uncastRCNode.(*kubegraph.ReplicationControllerNode)
for _, uncastPodNode := range g.PredecessorNodesByEdgeKind(rcNode, kubeedges.ManagedByRCEdgeKind) {
podNode := uncastPodNode.(*kubegraph.PodNode)
// check to see if this pod is managed by more than one RC
uncastOwningRCs := g.SuccessorNodesByEdgeKind(podNode, kubeedges.ManagedByRCEdgeKind)
if len(uncastOwningRCs) > 1 {
involvedRCNames := []string{}
relatedNodes := []graph.Node{uncastPodNode}
for _, uncastOwningRC := range uncastOwningRCs {
if uncastOwningRC.ID() == rcNode.ID() {
continue
}
owningRC := uncastOwningRC.(*kubegraph.ReplicationControllerNode)
involvedRCNames = append(involvedRCNames, f.ResourceName(owningRC))
relatedNodes = append(relatedNodes, uncastOwningRC)
}
markers = append(markers, osgraph.Marker{
Node: rcNode,
RelatedNodes: relatedNodes,
Severity: osgraph.WarningSeverity,
Key: DuelingReplicationControllerWarning,
Message: fmt.Sprintf("%s is competing for %s with %s", f.ResourceName(rcNode), f.ResourceName(podNode), strings.Join(involvedRCNames, ", ")),
})
}
}
}
return markers
}
func routePortMarker(g osgraph.Graph, f osgraph.Namer, routeNode *routegraph.RouteNode) *osgraph.Marker {
for _, uncastServiceNode := range g.SuccessorNodesByEdgeKind(routeNode, routeedges.ExposedThroughRouteEdgeKind) {
svcNode := uncastServiceNode.(*kubegraph.ServiceNode)
if !svcNode.Found() {
return &osgraph.Marker{
Node: routeNode,
RelatedNodes: []graph.Node{svcNode},
Severity: osgraph.WarningSeverity,
Key: MissingServiceWarning,
Message: fmt.Sprintf("%s is supposed to route traffic to %s but %s doesn't exist.",
f.ResourceName(routeNode), f.ResourceName(svcNode), f.ResourceName(svcNode)),
// TODO: Suggest 'oc create service' once that's a thing.
// See https://github.com/kubernetes/kubernetes/pull/19509
}
}
if len(svcNode.Spec.Ports) > 1 && (routeNode.Spec.Port == nil || len(routeNode.Spec.Port.TargetPort.String()) == 0) {
return &osgraph.Marker{
Node: routeNode,
RelatedNodes: []graph.Node{svcNode},
Severity: osgraph.WarningSeverity,
Key: MissingRoutePortWarning,
Message: fmt.Sprintf("%s doesn't have a port specified and is routing traffic to %s which uses multiple ports.",
f.ResourceName(routeNode), f.ResourceName(svcNode)),
}
}
if routeNode.Spec.Port == nil {
// If no port is specified, we don't need to analyze any further.
return nil
}
routePortString := routeNode.Spec.Port.TargetPort.String()
if routePort, err := strconv.Atoi(routePortString); err == nil {
for _, port := range svcNode.Spec.Ports {
if port.TargetPort.IntValue() == routePort {
return nil
}
}
// route has a numeric port, service has no port with that number as a targetPort.
marker := &osgraph.Marker{
Node: routeNode,
RelatedNodes: []graph.Node{svcNode},
Severity: osgraph.WarningSeverity,
Key: WrongRoutePortWarning,
Message: fmt.Sprintf("%s has a port specified (%d) but %s has no such targetPort.",
f.ResourceName(routeNode), routePort, f.ResourceName(svcNode)),
}
if len(svcNode.Spec.Ports) == 1 {
marker.Suggestion = osgraph.Suggestion(fmt.Sprintf("oc patch %s -p '{\"spec\":{\"port\":{\"targetPort\": %d}}}'", f.ResourceName(routeNode), svcNode.Spec.Ports[0].TargetPort.IntValue()))
}
return marker
}
for _, port := range svcNode.Spec.Ports {
if port.Name == routePortString {
return nil
}
}
// route has a named port, service has no port with that name.
marker := &osgraph.Marker{
Node: routeNode,
RelatedNodes: []graph.Node{svcNode},
Severity: osgraph.WarningSeverity,
Key: WrongRoutePortWarning,
Message: fmt.Sprintf("%s has a named port specified (%q) but %s has no such named port.",
f.ResourceName(routeNode), routePortString, f.ResourceName(svcNode)),
}
if len(svcNode.Spec.Ports) == 1 {
marker.Suggestion = osgraph.Suggestion(fmt.Sprintf("oc patch %s -p '{\"spec\":{\"port\":{\"targetPort\": %d}}}'", f.ResourceName(routeNode), svcNode.Spec.Ports[0].TargetPort.IntValue()))
}
return marker
}
return nil
}
func FindPathBasedPassthroughRoutes(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastRouteNode := range g.NodesByKind(routegraph.RouteNodeKind) {
routeNode := uncastRouteNode.(*routegraph.RouteNode)
if len(routeNode.Spec.Path) > 0 && routeNode.Spec.TLS != nil && routeNode.Spec.TLS.Termination == routeapi.TLSTerminationPassthrough {
markers = append(markers, osgraph.Marker{
Node: routeNode,
Severity: osgraph.ErrorSeverity,
Key: PathBasedPassthroughErr,
Message: fmt.Sprintf("%s is path-based and uses passthrough termination, which is an invalid combination.", f.ResourceName(routeNode)),
Suggestion: osgraph.Suggestion(fmt.Sprintf("1. use spec.tls.termination=edge or 2. use spec.tls.termination=reencrypt and specify spec.tls.destinationCACertificate or 3. remove spec.path")),
})
}
}
return markers
}
// getImageStreamImageMarker will return the appropriate marker for when a BuildConfig is missing its input ImageStreamImage
func getImageStreamImageMarker(g osgraph.Graph, f osgraph.Namer, bcNode graph.Node, bcInputNode graph.Node, imageStreamNode graph.Node, imageNode *imagegraph.ImageStreamImageNode, imageStream *imageapi.ImageStream, imageID string) osgraph.Marker {
return osgraph.Marker{
Node: bcNode,
RelatedNodes: []graph.Node{bcInputNode,
imageStreamNode},
Severity: osgraph.WarningSeverity,
Key: MissingImageStreamImageWarning,
Message: fmt.Sprintf("%s builds from %s, but the image stream image does not exist.", f.ResourceName(bcNode), f.ResourceName(bcInputNode)),
Suggestion: getImageStreamImageSuggestion(imageID, imageStream),
}
}
func FindMissingTLSTerminationType(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastRouteNode := range g.NodesByKind(routegraph.RouteNodeKind) {
routeNode := uncastRouteNode.(*routegraph.RouteNode)
if routeNode.Spec.TLS != nil && len(routeNode.Spec.TLS.Termination) == 0 {
markers = append(markers, osgraph.Marker{
Node: routeNode,
Severity: osgraph.ErrorSeverity,
Key: MissingTLSTerminationTypeErr,
Message: fmt.Sprintf("%s has a TLS configuration but no termination type specified.", f.ResourceName(routeNode)),
Suggestion: osgraph.Suggestion(fmt.Sprintf("oc patch %s -p '{\"spec\":{\"tls\":{\"termination\":\"<type>\"}}}' (replace <type> with a valid termination type: edge, passthrough, reencrypt)", f.ResourceName(routeNode)))})
}
}
return markers
}
// FindMissingRouter creates markers for all routes in case there is no running router.
func FindMissingRouter(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastRouteNode := range g.NodesByKind(routegraph.RouteNodeKind) {
routeNode := uncastRouteNode.(*routegraph.RouteNode)
if len(routeNode.Route.Status.Ingress) == 0 {
markers = append(markers, osgraph.Marker{
Node: routeNode,
Severity: osgraph.ErrorSeverity,
Key: MissingRequiredRouterErr,
Message: fmt.Sprintf("%s is routing traffic to svc/%s, but either the administrator has not installed a router or the router is not selecting this route.", f.ResourceName(routeNode), routeNode.Spec.To.Name),
Suggestion: osgraph.Suggestion("oc adm router -h"),
})
}
}
return markers
}
// FindRestartingPods inspects all Pods to see if they've restarted more than the threshold. logsCommandName is the name of
// the command that should be invoked to see pod logs. securityPolicyCommandPattern is a format string accepting two replacement
// variables for fmt.Sprintf - 1, the namespace of the current pod, 2 the service account of the pod.
func FindRestartingPods(g osgraph.Graph, f osgraph.Namer, logsCommandName, securityPolicyCommandPattern string) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, uncastPodNode := range g.NodesByKind(kubegraph.PodNodeKind) {
podNode := uncastPodNode.(*kubegraph.PodNode)
pod, ok := podNode.Object().(*kapi.Pod)
if !ok {
continue
}
for _, containerStatus := range pod.Status.ContainerStatuses {
switch {
case containerCrashLoopBackOff(containerStatus):
var suggestion string
switch {
case containerIsNonRoot(pod, containerStatus.Name):
suggestion = heredoc.Docf(`
The container is starting and exiting repeatedly. This usually means the container is unable
to start, misconfigured, or limited by security restrictions. Check the container logs with
%s %s -c %s
Current security policy prevents your containers from being run as the root user. Some images
may fail expecting to be able to change ownership or permissions on directories. Your admin
can grant you access to run containers that need to run as the root user with this command:
%s
`, logsCommandName, pod.Name, containerStatus.Name, fmt.Sprintf(securityPolicyCommandPattern, pod.Namespace, pod.Spec.ServiceAccountName))
default:
suggestion = heredoc.Docf(`
The container is starting and exiting repeatedly. This usually means the container is unable
to start, misconfigured, or limited by security restrictions. Check the container logs with
%s %s -c %s
`, logsCommandName, pod.Name, containerStatus.Name)
}
markers = append(markers, osgraph.Marker{
Node: podNode,
Severity: osgraph.ErrorSeverity,
Key: CrashLoopingPodError,
Message: fmt.Sprintf("container %q in %s is crash-looping", containerStatus.Name,
f.ResourceName(podNode)),
Suggestion: osgraph.Suggestion(suggestion),
})
case containerRestartedRecently(containerStatus, nowFn()):
markers = append(markers, osgraph.Marker{
Node: podNode,
Severity: osgraph.WarningSeverity,
Key: RestartingPodWarning,
Message: fmt.Sprintf("container %q in %s has restarted within the last 10 minutes", containerStatus.Name,
f.ResourceName(podNode)),
})
case containerRestartedFrequently(containerStatus):
markers = append(markers, osgraph.Marker{
Node: podNode,
Severity: osgraph.WarningSeverity,
Key: RestartingPodWarning,
Message: fmt.Sprintf("container %q in %s has restarted %d times", containerStatus.Name,
f.ResourceName(podNode), containerStatus.RestartCount),
})
}
}
}
return markers
}
// FindMissingInputImageStreams checks all build configs and confirms that their From element exists
//
// Precedence of failures:
// 1. A build config's input points to an image stream that does not exist
// 2. A build config's input uses an image stream tag reference in an existing image stream, but no images within the image stream have that tag assigned
// 3. A build config's input uses an image stream image reference in an exisiting image stream, but no images within the image stream have the supplied image hexadecimal ID
func FindMissingInputImageStreams(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
markers := []osgraph.Marker{}
for _, bcNode := range g.NodesByKind(buildgraph.BuildConfigNodeKind) {
for _, bcInputNode := range g.PredecessorNodesByEdgeKind(bcNode, buildedges.BuildInputImageEdgeKind) {
switch bcInputNode.(type) {
case *imagegraph.ImageStreamTagNode:
for _, uncastImageStreamNode := range g.SuccessorNodesByEdgeKind(bcInputNode, imageedges.ReferencedImageStreamGraphEdgeKind) {
imageStreamNode := uncastImageStreamNode.(*imagegraph.ImageStreamNode)
// note, BuildConfig.Spec.BuildSpec.Strategy.[Docker|Source|Custom]Stragegy.From Input of ImageStream has been converted to ImageStreamTag on the vX to api conversion
// prior to our reaching this point in the code; so there is not need to check for that type vs. ImageStreamTag or ImageStreamImage;
tagNode, _ := bcInputNode.(*imagegraph.ImageStreamTagNode)
imageStream := imageStreamNode.Object().(*imageapi.ImageStream)
if _, ok := imageStream.Status.Tags[tagNode.ImageTag()]; !ok {
markers = append(markers, osgraph.Marker{
Node: bcNode,
RelatedNodes: []graph.Node{bcInputNode,
imageStreamNode},
Severity: osgraph.WarningSeverity,
Key: MissingImageStreamTagWarning,
Message: fmt.Sprintf("%s builds from %s, but the image stream tag does not exist.", f.ResourceName(bcNode), f.ResourceName(bcInputNode)),
Suggestion: osgraph.Suggestion(fmt.Sprintf("examine analysis of build config outputs from this command and see if they build %s", f.ResourceName(bcInputNode))),
})
}
}
case *imagegraph.ImageStreamImageNode:
for _, uncastImageStreamNode := range g.SuccessorNodesByEdgeKind(bcInputNode, imageedges.ReferencedImageStreamImageGraphEdgeKind) {
imageStreamNode := uncastImageStreamNode.(*imagegraph.ImageStreamNode)
imageNode, _ := bcInputNode.(*imagegraph.ImageStreamImageNode)
imageStream := imageStreamNode.Object().(*imageapi.ImageStream)
found, imageID, suggestion := validImageStreamImage(imageNode, imageStream)
if !found {
markers = append(markers, osgraph.Marker{
Node: bcNode,
RelatedNodes: []graph.Node{bcInputNode,
imageStreamNode},
Severity: osgraph.WarningSeverity,
Key: MissingImageStreamImageWarning,
Message: fmt.Sprintf("%s builds from %s, but the image stream image does not exist.", f.ResourceName(bcNode), f.ResourceName(bcInputNode)),
Suggestion: osgraph.Suggestion(fmt.Sprintf(suggestion, imageID, f.ResourceName(imageStreamNode))),
})
}
}
}
}
}
return markers
}
