func printSingleResourceUsage(out io.Writer, resourceType api.ResourceName, quantity resource.Quantity) {
switch resourceType {
case api.ResourceCPU:
fmt.Fprintf(out, "%vm", quantity.MilliValue())
case api.ResourceMemory:
fmt.Fprintf(out, "%vMi", quantity.Value()/(1024*1024))
default:
fmt.Fprintf(out, "%v", quantity.Value())
}
}
// PodsRequests returns sum of each resource request for each pod in list
// If a given pod in the list does not have a request for the named resource, we log the error
// but still attempt to get the most representative count
func PodsRequests(pods []*api.Pod, resourceName api.ResourceName) *resource.Quantity {
var sum *resource.Quantity
for i := range pods {
pod := pods[i]
podQuantity, err := PodRequests(pod, resourceName)
if err != nil {
// log the error, but try to keep the most accurate count possible in log
// rationale here is that you may have had pods in a namespace that did not have
// explicit requests prior to adding the quota
glog.Infof("No explicit request for resource, pod %s/%s, %s", pod.Namespace, pod.Name, resourceName)
} else {
if sum == nil {
sum = podQuantity
} else {
sum.Add(*podQuantity)
}
}
}
// if list is empty
if sum == nil {
q := resource.MustParse("0")
sum = &q
}
return sum
}
func formatImageStreamQuota(out *tabwriter.Writer, c client.Interface, kc kclient.Interface, stream *imageapi.ImageStream) {
quotas, err := kc.ResourceQuotas(stream.Namespace).List(api.ListOptions{})
if err != nil {
return
}
var limit *resource.Quantity
for _, item := range quotas.Items {
// search for smallest ImageStream quota
if value, ok := item.Spec.Hard[imageapi.ResourceImageStreamSize]; ok {
if limit == nil || limit.Cmp(value) > 0 {
limit = &value
}
}
}
if limit != nil {
quantity := imagequota.GetImageStreamSize(c, stream, make(map[string]*imageapi.Image))
scale := mega
if quantity.Value() >= (1<<giga.scale) || limit.Value() >= (1<<giga.scale) {
scale = giga
}
formatString(out, "Quota Usage", fmt.Sprintf("%s / %s",
formatQuantity(quantity, scale), formatQuantity(limit, scale)))
}
}
// Validates that a Quantity is not negative
func ValidateNonnegativeQuantity(value resource.Quantity, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if value.Cmp(resource.Quantity{}) < 0 {
allErrs = append(allErrs, field.Invalid(fldPath, value.String(), isNegativeErrorMsg))
}
return allErrs
}
// Calculates score for all pods and returns podInfo structure.
// Score is defined as cpu_sum/node_capacity + mem_sum/node_capacity.
// Pods that have bigger requirements should be processed first, thus have higher scores.
func calculatePodScore(pods []*apiv1.Pod, nodeTemplate *schedulercache.NodeInfo) []*podInfo {
podInfos := make([]*podInfo, 0, len(pods))
for _, pod := range pods {
cpuSum := resource.Quantity{}
memorySum := resource.Quantity{}
for _, container := range pod.Spec.Containers {
if request, ok := container.Resources.Requests[apiv1.ResourceCPU]; ok {
cpuSum.Add(request)
}
if request, ok := container.Resources.Requests[apiv1.ResourceMemory]; ok {
memorySum.Add(request)
}
}
score := float64(0)
if cpuAllocatable, ok := nodeTemplate.Node().Status.Allocatable[apiv1.ResourceCPU]; ok && cpuAllocatable.MilliValue() > 0 {
score += float64(cpuSum.MilliValue()) / float64(cpuAllocatable.MilliValue())
}
if memAllocatable, ok := nodeTemplate.Node().Status.Allocatable[apiv1.ResourceMemory]; ok && memAllocatable.Value() > 0 {
score += float64(memorySum.Value()) / float64(memAllocatable.Value())
}
podInfos = append(podInfos, &podInfo{
score: score,
pod: pod,
})
}
return podInfos
}
func validateBandwidthIsReasonable(rsrc *resource.Quantity) error {
if rsrc.Value() < minRsrc.Value() {
return fmt.Errorf("resource is unreasonably small (< 1kbit)")
}
if rsrc.Value() > maxRsrc.Value() {
return fmt.Errorf("resoruce is unreasonably large (> 1Pbit)")
}
return nil
}
// ValidateResourceQuantityValue enforces that specified quantity is valid for specified resource
func ValidateResourceQuantityValue(resource string, value resource.Quantity, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
allErrs = append(allErrs, ValidateNonnegativeQuantity(value, fldPath)...)
if api.IsIntegerResourceName(resource) {
if value.MilliValue()%int64(1000) != int64(0) {
allErrs = append(allErrs, field.Invalid(fldPath, value, isNotIntegerErrorMsg))
}
}
return allErrs
}
// The method veryfies whether resources should be set for the given pod and
// if there is estimation available the method fills Request field.
func (ir initialResources) estimateAndFillResourcesIfNotSet(pod *api.Pod) {
annotations := []string{}
for i := range pod.Spec.Containers {
c := &pod.Spec.Containers[i]
req := c.Resources.Requests
lim := c.Resources.Limits
var cpu, mem *resource.Quantity
var err error
if _, ok := req[api.ResourceCPU]; !ok {
if _, ok2 := lim[api.ResourceCPU]; !ok2 {
cpu, err = ir.getEstimation(api.ResourceCPU, c, pod.ObjectMeta.Namespace)
if err != nil {
glog.Errorf("Error while trying to estimate resources: %v", err)
}
}
}
if _, ok := req[api.ResourceMemory]; !ok {
if _, ok2 := lim[api.ResourceMemory]; !ok2 {
mem, err = ir.getEstimation(api.ResourceMemory, c, pod.ObjectMeta.Namespace)
if err != nil {
glog.Errorf("Error while trying to estimate resources: %v", err)
}
}
}
// If Requests doesn't exits and an estimation was made, create Requests.
if req == nil && (cpu != nil || mem != nil) {
c.Resources.Requests = api.ResourceList{}
req = c.Resources.Requests
}
setRes := []string{}
if cpu != nil {
glog.Infof("CPU estimation for container %v in pod %v/%v is %v", c.Name, pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, cpu.String())
setRes = append(setRes, string(api.ResourceCPU))
req[api.ResourceCPU] = *cpu
}
if mem != nil {
glog.Infof("Memory estimation for container %v in pod %v/%v is %v", c.Name, pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, mem.String())
setRes = append(setRes, string(api.ResourceMemory))
req[api.ResourceMemory] = *mem
}
if len(setRes) > 0 {
sort.Strings(setRes)
a := strings.Join(setRes, ", ") + " request for container " + c.Name
annotations = append(annotations, a)
}
}
if len(annotations) > 0 {
if pod.ObjectMeta.Annotations == nil {
pod.ObjectMeta.Annotations = make(map[string]string)
}
val := "Initial Resources plugin set: " + strings.Join(annotations, "; ")
pod.ObjectMeta.Annotations[initialResourcesAnnotation] = val
}
}
// formatQuantity prints quantity according to passed scale. Manual scaling was
// done here to make sure we print correct binary values for quantity.
func formatQuantity(quantity *resource.Quantity, scale scale) string {
integer := quantity.Value() >> scale.scale
// fraction is the reminder of a division shifted by one order of magnitude
fraction := (quantity.Value() % (1 << scale.scale)) >> (scale.scale - 10)
// additionally we present only 2 digits after dot, so divide by 10
fraction = fraction / 10
if fraction > 0 {
return fmt.Sprintf("%d.%02d%s", integer, fraction, scale.unit)
}
return fmt.Sprintf("%d%s", integer, scale.unit)
}
func deepCopy_resource_Quantity(in resource.Quantity, out *resource.Quantity, c *conversion.Cloner) error {
if in.Amount != nil {
if newVal, err := c.DeepCopy(in.Amount); err != nil {
return err
} else {
out.Amount = newVal.(*inf.Dec)
}
} else {
out.Amount = nil
}
out.Format = in.Format
return nil
}
func (cr *realQuotaCommandRunner) RunApplyQuotaCommand(fsDevice string, quota resource.Quantity, fsGroup int64) (string, string, error) {
args := []string{"-x", "-c",
fmt.Sprintf("limit -g bsoft=%d bhard=%d %d", quota.Value(), quota.Value(), fsGroup),
fsDevice,
}
cmd := exec.Command("xfs_quota", args...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
err := cmd.Run()
glog.V(5).Infof("Ran: xfs_quota %s", args)
return "", stderr.String(), err
}
func (xqa *xfsQuotaApplicator) applyQuota(volDevice string, quota resource.Quantity, fsGroupID int64) error {
_, stderr, err := xqa.cmdRunner.RunApplyQuotaCommand(volDevice, quota, fsGroupID)
if err != nil {
return err
}
// xfs_quota is very happy to fail but return a success code, likely due to its
// interactive shell approach. Grab stderr, if we see anything written to it we'll
// consider this an error.
if len(stderr) > 0 {
return fmt.Errorf("xfs_quota wrote to stderr: %s", stderr)
}
glog.V(4).Infof("XFS quota applied: device=%s, quota=%d, fsGroup=%d", volDevice, quota.Value(), fsGroupID)
return nil
}
func getScheduableCores(nodes []v1.Node) int64 {
var sc resource.Quantity
for _, node := range nodes {
if !node.Spec.Unschedulable {
sc.Add(node.Status.Capacity[v1.ResourceCPU])
}
}
scInt64, scOk := sc.AsInt64()
if !scOk {
framework.Logf("Unable to compute integer values of schedulable cores in the cluster")
return 0
}
return scInt64
}
func newPodMemoryStats(pod *api.Pod, workingSet resource.Quantity) statsapi.PodStats {
result := statsapi.PodStats{
PodRef: statsapi.PodReference{
Name: pod.Name, Namespace: pod.Namespace, UID: string(pod.UID),
},
}
for range pod.Spec.Containers {
workingSetBytes := uint64(workingSet.Value())
result.Containers = append(result.Containers, statsapi.ContainerStats{
Memory: &statsapi.MemoryStats{
WorkingSetBytes: &workingSetBytes,
},
})
}
return result
}
func (ir initialResources) estimateContainer(pod *api.Pod, c *api.Container, message string) []string {
var annotations []string
req := c.Resources.Requests
lim := c.Resources.Limits
var cpu, mem *resource.Quantity
var err error
if _, ok := req[api.ResourceCPU]; !ok {
if _, ok2 := lim[api.ResourceCPU]; !ok2 {
cpu, err = ir.getEstimation(api.ResourceCPU, c, pod.ObjectMeta.Namespace)
if err != nil {
glog.Errorf("Error while trying to estimate resources: %v", err)
}
}
}
if _, ok := req[api.ResourceMemory]; !ok {
if _, ok2 := lim[api.ResourceMemory]; !ok2 {
mem, err = ir.getEstimation(api.ResourceMemory, c, pod.ObjectMeta.Namespace)
if err != nil {
glog.Errorf("Error while trying to estimate resources: %v", err)
}
}
}
// If Requests doesn't exits and an estimation was made, create Requests.
if req == nil && (cpu != nil || mem != nil) {
c.Resources.Requests = api.ResourceList{}
req = c.Resources.Requests
}
setRes := []string{}
if cpu != nil {
glog.Infof("CPU estimation for %s %v in pod %v/%v is %v", message, c.Name, pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, cpu.String())
setRes = append(setRes, string(api.ResourceCPU))
req[api.ResourceCPU] = *cpu
}
if mem != nil {
glog.Infof("Memory estimation for %s %v in pod %v/%v is %v", message, c.Name, pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, mem.String())
setRes = append(setRes, string(api.ResourceMemory))
req[api.ResourceMemory] = *mem
}
if len(setRes) > 0 {
sort.Strings(setRes)
a := strings.Join(setRes, ", ") + fmt.Sprintf(" request for %s %s", message, c.Name)
annotations = append(annotations, a)
}
return annotations
}
// podDiskUsage aggregates pod disk usage for the specified stats to measure.
func podDiskUsage(podStats statsapi.PodStats, pod *api.Pod, statsToMeasure []fsStatsType) (api.ResourceList, error) {
disk := resource.Quantity{Format: resource.BinarySI}
for _, container := range podStats.Containers {
if hasFsStatsType(statsToMeasure, fsStatsRoot) {
disk.Add(*diskUsage(container.Rootfs))
}
if hasFsStatsType(statsToMeasure, fsStatsLogs) {
disk.Add(*diskUsage(container.Logs))
}
}
if hasFsStatsType(statsToMeasure, fsStatsLocalVolumeSource) {
volumeNames := localVolumeNames(pod)
for _, volumeName := range volumeNames {
for _, volumeStats := range podStats.VolumeStats {
if volumeStats.Name == volumeName {
disk.Add(*diskUsage(&volumeStats.FsStats))
break
}
}
}
}
return api.ResourceList{
resourceDisk: disk,
}, nil
}
func (xqa *xfsQuotaApplicator) applyQuota(volDevice string, quota resource.Quantity, fsGroupID int64) error {
_, stderr, err := xqa.cmdRunner.RunApplyQuotaCommand(volDevice, quota, fsGroupID)
// xfs_quota is very happy to fail but return a success code, likely due to its
// interactive shell approach. Grab stderr, if we see anything written to it we'll
// consider this an error.
//
// If we exit non-zero *and* write to stderr, stderr is likely to have the details on what
// actually went wrong, so we'll use this as the error message instead.
if len(stderr) > 0 {
return fmt.Errorf("error applying quota: %s", stderr)
}
if err != nil {
return fmt.Errorf("error applying quota: %v", err)
}
glog.V(4).Infof("XFS quota applied: device=%s, quota=%d, fsGroup=%d", volDevice, quota.Value(), fsGroupID)
return nil
}
// TODO: remove this duplicate
// InternalExtractContainerResourceValue extracts the value of a resource
// in an already known container
func InternalExtractContainerResourceValue(fs *api.ResourceFieldSelector, container *api.Container) (string, error) {
divisor := resource.Quantity{}
if divisor.Cmp(fs.Divisor) == 0 {
divisor = resource.MustParse("1")
} else {
divisor = fs.Divisor
}
switch fs.Resource {
case "limits.cpu":
return convertResourceCPUToString(container.Resources.Limits.Cpu(), divisor)
case "limits.memory":
return convertResourceMemoryToString(container.Resources.Limits.Memory(), divisor)
case "requests.cpu":
return convertResourceCPUToString(container.Resources.Requests.Cpu(), divisor)
case "requests.memory":
return convertResourceMemoryToString(container.Resources.Requests.Memory(), divisor)
}
return "", fmt.Errorf("Unsupported container resource : %v", fs.Resource)
}
// newPodDiskStats returns stats with specified usage amounts.
func newPodDiskStats(pod *api.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats {
result := statsapi.PodStats{
PodRef: statsapi.PodReference{
Name: pod.Name, Namespace: pod.Namespace, UID: string(pod.UID),
},
}
rootFsUsedBytes := uint64(rootFsUsed.Value())
logsUsedBytes := uint64(logsUsed.Value())
for range pod.Spec.Containers {
result.Containers = append(result.Containers, statsapi.ContainerStats{
Rootfs: &statsapi.FsStats{
UsedBytes: &rootFsUsedBytes,
},
Logs: &statsapi.FsStats{
UsedBytes: &logsUsedBytes,
},
})
}
perLocalVolumeUsedBytes := uint64(perLocalVolumeUsed.Value())
for _, volumeName := range localVolumeNames(pod) {
result.VolumeStats = append(result.VolumeStats, statsapi.VolumeStats{
Name: volumeName,
FsStats: statsapi.FsStats{
UsedBytes: &perLocalVolumeUsedBytes,
},
})
}
return result
}
// The method veryfies whether resources should be set for the given pod and
// if there is estimation available the method fills Request field.
func (ir initialResources) estimateAndFillResourcesIfNotSet(pod *api.Pod) {
for i := range pod.Spec.Containers {
c := &pod.Spec.Containers[i]
req := c.Resources.Requests
lim := c.Resources.Limits
var cpu, mem *resource.Quantity
var err error
if _, ok := req[api.ResourceCPU]; !ok {
if _, ok2 := lim[api.ResourceCPU]; !ok2 {
cpu, err = ir.getEstimation(api.ResourceCPU, c)
if err != nil {
glog.Errorf("Error while trying to estimate resources: %v", err)
}
}
}
if _, ok := req[api.ResourceMemory]; !ok {
if _, ok2 := lim[api.ResourceMemory]; !ok2 {
mem, err = ir.getEstimation(api.ResourceMemory, c)
if err != nil {
glog.Errorf("Error while trying to estimate resources: %v", err)
}
}
}
// If Requests doesn't exits and an estimation was made, create Requests.
if req == nil && (cpu != nil || mem != nil) {
c.Resources.Requests = api.ResourceList{}
req = c.Resources.Requests
}
if cpu != nil {
glog.Infof("CPU estimation for container %v in pod %v/%v is %v", c.Name, pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, cpu.String())
req[api.ResourceCPU] = *cpu
}
if mem != nil {
glog.Infof("Memory estimation for container %v in pod %v/%v is %v", c.Name, pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, mem.String())
req[api.ResourceMemory] = *mem
}
}
// TODO(piosz): verify the estimates fits in LimitRanger
}
// limitRequestRatioConstraint enforces the limit to request ratio over the specified resource
func limitRequestRatioConstraint(limitType api.LimitType, resourceName api.ResourceName, enforced resource.Quantity, request api.ResourceList, limit api.ResourceList) error {
req, reqExists := request[resourceName]
lim, limExists := limit[resourceName]
observedReqValue, observedLimValue, _ := requestLimitEnforcedValues(req, lim, enforced)
if !reqExists || (observedReqValue == int64(0)) {
return fmt.Errorf("%s max limit to request ratio per %s is %s, but no request is specified or request is 0.", resourceName, limitType, enforced.String())
}
if !limExists || (observedLimValue == int64(0)) {
return fmt.Errorf("%s max limit to request ratio per %s is %s, but no limit is specified or limit is 0.", resourceName, limitType, enforced.String())
}
observedRatio := float64(observedLimValue) / float64(observedReqValue)
displayObservedRatio := observedRatio
maxLimitRequestRatio := float64(enforced.Value())
if enforced.Value() <= resource.MaxMilliValue {
observedRatio = observedRatio * 1000
maxLimitRequestRatio = float64(enforced.MilliValue())
}
if observedRatio > maxLimitRequestRatio {
return fmt.Errorf("%s max limit to request ratio per %s is %s, but provided ratio is %f.", resourceName, limitType, enforced.String(), displayObservedRatio)
}
return nil
}
func addCpuLimit(opts []*unit.UnitOption, limit *resource.Quantity) ([]*unit.UnitOption, error) {
if limit.Value() > resource.MaxMilliValue {
return nil, fmt.Errorf("cpu limit exceeds the maximum millivalue: %v", limit.String())
}
quota := strconv.Itoa(int(limit.MilliValue()/10)) + "%"
opts = append(opts, unit.NewUnitOption("Service", "CPUQuota", quota))
return opts, nil
}
// PodRequests returns sum of each resource request across all containers in pod
func PodRequests(pod *api.Pod, resourceName api.ResourceName) (*resource.Quantity, error) {
if !PodHasRequests(pod, resourceName) {
return nil, fmt.Errorf("Each container in pod %s/%s does not have an explicit request for resource %s.", pod.Namespace, pod.Name, resourceName)
}
var sum *resource.Quantity
for j := range pod.Spec.Containers {
value, _ := pod.Spec.Containers[j].Resources.Requests[resourceName]
if sum == nil {
sum = value.Copy()
} else {
err := sum.Add(value)
if err != nil {
return sum, err
}
}
}
// if list is empty
if sum == nil {
q := resource.MustParse("0")
sum = &q
}
return sum, nil
}
func printSingleResourceUsage(out io.Writer, resourceType v1.ResourceName, quantity resource.Quantity) {
switch resourceType {
case v1.ResourceCPU:
fmt.Fprintf(out, "%vm", quantity.MilliValue())
case v1.ResourceMemory:
fmt.Fprintf(out, "%vMi", quantity.Value()/(1024*1024))
case v1.ResourceStorage:
// TODO: Change it after storage metrics collection is finished.
fmt.Fprint(out, "-")
default:
fmt.Fprintf(out, "%v", quantity.Value())
}
}
// podMemoryUsage aggregates pod memory usage.
func podMemoryUsage(podStats statsapi.PodStats) (api.ResourceList, error) {
disk := resource.Quantity{Format: resource.BinarySI}
memory := resource.Quantity{Format: resource.BinarySI}
for _, container := range podStats.Containers {
// disk usage (if known)
for _, fsStats := range []*statsapi.FsStats{container.Rootfs, container.Logs} {
disk.Add(*diskUsage(fsStats))
}
// memory usage (if known)
memory.Add(*memoryUsage(container.Memory))
}
return api.ResourceList{
api.ResourceMemory: memory,
resourceDisk: disk,
}, nil
}
// podUsage aggregates usage of compute resources.
// it supports the following memory and disk.
func podUsage(podStats statsapi.PodStats) (api.ResourceList, error) {
disk := resource.Quantity{Format: resource.BinarySI}
memory := resource.Quantity{Format: resource.BinarySI}
for _, container := range podStats.Containers {
// disk usage (if known)
// TODO: need to handle volumes
for _, fsStats := range []*statsapi.FsStats{container.Rootfs, container.Logs} {
if err := disk.Add(*diskUsage(fsStats)); err != nil {
return nil, err
}
}
// memory usage (if known)
if err := memory.Add(*memoryUsage(container.Memory)); err != nil {
return nil, err
}
}
return api.ResourceList{
api.ResourceMemory: memory,
resourceDisk: disk,
}, nil
}
// FuzzerFor can randomly populate api objects that are destined for version.
func FuzzerFor(t *testing.T, version unversioned.GroupVersion, src rand.Source) *fuzz.Fuzzer {
f := fuzz.New().NilChance(.5).NumElements(1, 1)
if src != nil {
f.RandSource(src)
}
f.Funcs(
func(j *int, c fuzz.Continue) {
*j = int(c.Int31())
},
func(j **int, c fuzz.Continue) {
if c.RandBool() {
i := int(c.Int31())
*j = &i
} else {
*j = nil
}
},
func(q *resource.Quantity, c fuzz.Continue) {
*q = *resource.NewQuantity(c.Int63n(1000), resource.DecimalExponent)
},
func(j *runtime.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *unversioned.TypeMeta, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = ""
j.Kind = ""
},
func(j *api.ObjectMeta, c fuzz.Continue) {
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
j.UID = types.UID(c.RandString())
j.GenerateName = c.RandString()
var sec, nsec int64
c.Fuzz(&sec)
c.Fuzz(&nsec)
j.CreationTimestamp = unversioned.Unix(sec, nsec).Rfc3339Copy()
},
func(j *api.ObjectReference, c fuzz.Continue) {
// We have to customize the randomization of TypeMetas because their
// APIVersion and Kind must remain blank in memory.
j.APIVersion = c.RandString()
j.Kind = c.RandString()
j.Namespace = c.RandString()
j.Name = c.RandString()
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.FieldPath = c.RandString()
},
func(j *unversioned.ListMeta, c fuzz.Continue) {
j.ResourceVersion = strconv.FormatUint(c.RandUint64(), 10)
j.SelfLink = c.RandString()
},
func(j *api.ListOptions, c fuzz.Continue) {
label, _ := labels.Parse("a=b")
j.LabelSelector = label
field, _ := fields.ParseSelector("a=b")
j.FieldSelector = field
},
func(j *api.PodExecOptions, c fuzz.Continue) {
j.Stdout = true
j.Stderr = true
},
func(j *api.PodAttachOptions, c fuzz.Continue) {
j.Stdout = true
j.Stderr = true
},
func(s *api.PodSpec, c fuzz.Continue) {
c.FuzzNoCustom(s)
// has a default value
ttl := int64(30)
if c.RandBool() {
ttl = int64(c.Uint32())
}
s.TerminationGracePeriodSeconds = &ttl
c.Fuzz(s.SecurityContext)
if s.SecurityContext == nil {
s.SecurityContext = new(api.PodSecurityContext)
}
},
func(j *api.PodPhase, c fuzz.Continue) {
statuses := []api.PodPhase{api.PodPending, api.PodRunning, api.PodFailed, api.PodUnknown}
*j = statuses[c.Rand.Intn(len(statuses))]
},
func(j *api.Binding, c fuzz.Continue) {
c.Fuzz(&j.ObjectMeta)
j.Target.Name = c.RandString()
},
func(j *api.ReplicationControllerSpec, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
//j.TemplateRef = nil // this is required for round trip
},
func(j *extensions.DeploymentStrategy, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
// Ensure that strategyType is one of valid values.
strategyTypes := []extensions.DeploymentStrategyType{extensions.RecreateDeploymentStrategyType, extensions.RollingUpdateDeploymentStrategyType}
j.Type = strategyTypes[c.Rand.Intn(len(strategyTypes))]
if j.Type != extensions.RollingUpdateDeploymentStrategyType {
j.RollingUpdate = nil
} else {
rollingUpdate := extensions.RollingUpdateDeployment{}
if c.RandBool() {
rollingUpdate.MaxUnavailable = intstr.FromInt(int(c.RandUint64()))
rollingUpdate.MaxSurge = intstr.FromInt(int(c.RandUint64()))
} else {
rollingUpdate.MaxSurge = intstr.FromString(fmt.Sprintf("%d%%", c.RandUint64()))
}
j.RollingUpdate = &rollingUpdate
}
},
func(j *batch.JobSpec, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
completions := int32(c.Rand.Int31())
parallelism := int32(c.Rand.Int31())
j.Completions = &completions
j.Parallelism = ¶llelism
if c.Rand.Int31()%2 == 0 {
j.ManualSelector = newBool(true)
} else {
j.ManualSelector = nil
}
},
func(j *api.List, c fuzz.Continue) {
c.FuzzNoCustom(j) // fuzz self without calling this function again
// TODO: uncomment when round trip starts from a versioned object
if false { //j.Items == nil {
j.Items = []runtime.Object{}
}
},
func(j *runtime.Object, c fuzz.Continue) {
// TODO: uncomment when round trip starts from a versioned object
if true { //c.RandBool() {
*j = &runtime.Unknown{
// We do not set TypeMeta here because it is not carried through a round trip
Raw: []byte(`{"apiVersion":"unknown.group/unknown","kind":"Something","someKey":"someValue"}`),
ContentType: runtime.ContentTypeJSON,
}
} else {
types := []runtime.Object{&api.Pod{}, &api.ReplicationController{}}
t := types[c.Rand.Intn(len(types))]
c.Fuzz(t)
*j = t
}
},
func(q *api.ResourceRequirements, c fuzz.Continue) {
randomQuantity := func() resource.Quantity {
var q resource.Quantity
c.Fuzz(&q)
return q
}
q.Limits = make(api.ResourceList)
q.Requests = make(api.ResourceList)
cpuLimit := randomQuantity()
q.Limits[api.ResourceCPU] = *cpuLimit.Copy()
q.Requests[api.ResourceCPU] = *cpuLimit.Copy()
memoryLimit := randomQuantity()
q.Limits[api.ResourceMemory] = *memoryLimit.Copy()
q.Requests[api.ResourceMemory] = *memoryLimit.Copy()
storageLimit := randomQuantity()
q.Limits[api.ResourceStorage] = *storageLimit.Copy()
q.Requests[api.ResourceStorage] = *storageLimit.Copy()
},
func(q *api.LimitRangeItem, c fuzz.Continue) {
var cpuLimit resource.Quantity
c.Fuzz(&cpuLimit)
q.Type = api.LimitTypeContainer
q.Default = make(api.ResourceList)
q.Default[api.ResourceCPU] = *(cpuLimit.Copy())
q.DefaultRequest = make(api.ResourceList)
q.DefaultRequest[api.ResourceCPU] = *(cpuLimit.Copy())
q.Max = make(api.ResourceList)
q.Max[api.ResourceCPU] = *(cpuLimit.Copy())
q.Min = make(api.ResourceList)
q.Min[api.ResourceCPU] = *(cpuLimit.Copy())
q.MaxLimitRequestRatio = make(api.ResourceList)
q.MaxLimitRequestRatio[api.ResourceCPU] = resource.MustParse("10")
},
func(p *api.PullPolicy, c fuzz.Continue) {
policies := []api.PullPolicy{api.PullAlways, api.PullNever, api.PullIfNotPresent}
*p = policies[c.Rand.Intn(len(policies))]
},
func(rp *api.RestartPolicy, c fuzz.Continue) {
policies := []api.RestartPolicy{api.RestartPolicyAlways, api.RestartPolicyNever, api.RestartPolicyOnFailure}
*rp = policies[c.Rand.Intn(len(policies))]
},
// Only api.DownwardAPIVolumeFile needs to have a specific func since FieldRef has to be
// defaulted to a version otherwise roundtrip will fail
// For the remaining volume plugins the default fuzzer is enough.
func(m *api.DownwardAPIVolumeFile, c fuzz.Continue) {
m.Path = c.RandString()
versions := []string{"v1"}
m.FieldRef.APIVersion = versions[c.Rand.Intn(len(versions))]
m.FieldRef.FieldPath = c.RandString()
},
func(vs *api.VolumeSource, c fuzz.Continue) {
// Exactly one of the fields must be set.
v := reflect.ValueOf(vs).Elem()
i := int(c.RandUint64() % uint64(v.NumField()))
t := v.Field(i).Addr()
for v.Field(i).IsNil() {
c.Fuzz(t.Interface())
}
},
func(i *api.ISCSIVolumeSource, c fuzz.Continue) {
i.ISCSIInterface = c.RandString()
if i.ISCSIInterface == "" {
i.ISCSIInterface = "default"
}
},
func(d *api.DNSPolicy, c fuzz.Continue) {
policies := []api.DNSPolicy{api.DNSClusterFirst, api.DNSDefault}
*d = policies[c.Rand.Intn(len(policies))]
},
func(p *api.Protocol, c fuzz.Continue) {
protocols := []api.Protocol{api.ProtocolTCP, api.ProtocolUDP}
*p = protocols[c.Rand.Intn(len(protocols))]
},
func(p *api.ServiceAffinity, c fuzz.Continue) {
types := []api.ServiceAffinity{api.ServiceAffinityClientIP, api.ServiceAffinityNone}
*p = types[c.Rand.Intn(len(types))]
},
func(p *api.ServiceType, c fuzz.Continue) {
types := []api.ServiceType{api.ServiceTypeClusterIP, api.ServiceTypeNodePort, api.ServiceTypeLoadBalancer}
*p = types[c.Rand.Intn(len(types))]
},
func(ct *api.Container, c fuzz.Continue) {
c.FuzzNoCustom(ct) // fuzz self without calling this function again
ct.TerminationMessagePath = "/" + ct.TerminationMessagePath // Must be non-empty
},
func(p *api.Probe, c fuzz.Continue) {
c.FuzzNoCustom(p)
// These fields have default values.
intFieldsWithDefaults := [...]string{"TimeoutSeconds", "PeriodSeconds", "SuccessThreshold", "FailureThreshold"}
v := reflect.ValueOf(p).Elem()
for _, field := range intFieldsWithDefaults {
f := v.FieldByName(field)
if f.Int() == 0 {
f.SetInt(1)
}
}
},
func(ev *api.EnvVar, c fuzz.Continue) {
ev.Name = c.RandString()
if c.RandBool() {
ev.Value = c.RandString()
} else {
ev.ValueFrom = &api.EnvVarSource{}
ev.ValueFrom.FieldRef = &api.ObjectFieldSelector{}
var versions []unversioned.GroupVersion
for _, testGroup := range testapi.Groups {
versions = append(versions, *testGroup.GroupVersion())
}
ev.ValueFrom.FieldRef.APIVersion = versions[c.Rand.Intn(len(versions))].String()
ev.ValueFrom.FieldRef.FieldPath = c.RandString()
}
},
func(sc *api.SecurityContext, c fuzz.Continue) {
c.FuzzNoCustom(sc) // fuzz self without calling this function again
if c.RandBool() {
priv := c.RandBool()
sc.Privileged = &priv
}
if c.RandBool() {
sc.Capabilities = &api.Capabilities{
Add: make([]api.Capability, 0),
Drop: make([]api.Capability, 0),
}
c.Fuzz(&sc.Capabilities.Add)
c.Fuzz(&sc.Capabilities.Drop)
}
},
func(s *api.Secret, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
s.Type = api.SecretTypeOpaque
},
func(pv *api.PersistentVolume, c fuzz.Continue) {
c.FuzzNoCustom(pv) // fuzz self without calling this function again
types := []api.PersistentVolumePhase{api.VolumeAvailable, api.VolumePending, api.VolumeBound, api.VolumeReleased, api.VolumeFailed}
pv.Status.Phase = types[c.Rand.Intn(len(types))]
pv.Status.Message = c.RandString()
reclamationPolicies := []api.PersistentVolumeReclaimPolicy{api.PersistentVolumeReclaimRecycle, api.PersistentVolumeReclaimRetain}
pv.Spec.PersistentVolumeReclaimPolicy = reclamationPolicies[c.Rand.Intn(len(reclamationPolicies))]
},
func(pvc *api.PersistentVolumeClaim, c fuzz.Continue) {
c.FuzzNoCustom(pvc) // fuzz self without calling this function again
types := []api.PersistentVolumeClaimPhase{api.ClaimBound, api.ClaimPending}
pvc.Status.Phase = types[c.Rand.Intn(len(types))]
},
func(s *api.NamespaceSpec, c fuzz.Continue) {
s.Finalizers = []api.FinalizerName{api.FinalizerKubernetes}
},
func(s *api.NamespaceStatus, c fuzz.Continue) {
s.Phase = api.NamespaceActive
},
func(http *api.HTTPGetAction, c fuzz.Continue) {
c.FuzzNoCustom(http) // fuzz self without calling this function again
http.Path = "/" + http.Path // can't be blank
http.Scheme = "x" + http.Scheme // can't be blank
},
func(ss *api.ServiceSpec, c fuzz.Continue) {
c.FuzzNoCustom(ss) // fuzz self without calling this function again
if len(ss.Ports) == 0 {
// There must be at least 1 port.
ss.Ports = append(ss.Ports, api.ServicePort{})
c.Fuzz(&ss.Ports[0])
}
for i := range ss.Ports {
switch ss.Ports[i].TargetPort.Type {
case intstr.Int:
ss.Ports[i].TargetPort.IntVal = 1 + ss.Ports[i].TargetPort.IntVal%65535 // non-zero
case intstr.String:
ss.Ports[i].TargetPort.StrVal = "x" + ss.Ports[i].TargetPort.StrVal // non-empty
}
}
},
func(n *api.Node, c fuzz.Continue) {
c.FuzzNoCustom(n)
n.Spec.ExternalID = "external"
},
func(s *api.NodeStatus, c fuzz.Continue) {
c.FuzzNoCustom(s)
s.Allocatable = s.Capacity
},
func(s *extensions.HorizontalPodAutoscalerSpec, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
minReplicas := int32(c.Rand.Int31())
s.MinReplicas = &minReplicas
s.CPUUtilization = &extensions.CPUTargetUtilization{TargetPercentage: int32(c.RandUint64())}
},
func(s *extensions.SubresourceReference, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
s.Subresource = "scale"
},
func(psp *extensions.PodSecurityPolicySpec, c fuzz.Continue) {
c.FuzzNoCustom(psp) // fuzz self without calling this function again
runAsUserRules := []extensions.RunAsUserStrategy{extensions.RunAsUserStrategyMustRunAsNonRoot, extensions.RunAsUserStrategyMustRunAs, extensions.RunAsUserStrategyRunAsAny}
psp.RunAsUser.Rule = runAsUserRules[c.Rand.Intn(len(runAsUserRules))]
seLinuxRules := []extensions.SELinuxStrategy{extensions.SELinuxStrategyRunAsAny, extensions.SELinuxStrategyMustRunAs}
psp.SELinux.Rule = seLinuxRules[c.Rand.Intn(len(seLinuxRules))]
},
func(s *extensions.Scale, c fuzz.Continue) {
c.FuzzNoCustom(s) // fuzz self without calling this function again
// TODO: Implement a fuzzer to generate valid keys, values and operators for
// selector requirements.
if s.Status.Selector != nil {
s.Status.Selector = &unversioned.LabelSelector{
MatchLabels: map[string]string{
"testlabelkey": "testlabelval",
},
MatchExpressions: []unversioned.LabelSelectorRequirement{
{
Key: "testkey",
Operator: unversioned.LabelSelectorOpIn,
Values: []string{"val1", "val2", "val3"},
},
},
}
}
},
)
return f
}
func Convert_resource_Quantity_To_resource_Quantity(in *resource.Quantity, out *resource.Quantity, s conversion.Scope) error {
// Cannot deep copy these, because inf.Dec has unexported fields.
*out = *in.Copy()
return nil
}
// syncResourceQuota runs a complete sync of current status
func (rq *ResourceQuotaController) syncResourceQuota(quota api.ResourceQuota) (err error) {
// quota is dirty if any part of spec hard limits differs from the status hard limits
dirty := !api.Semantic.DeepEqual(quota.Spec.Hard, quota.Status.Hard)
// dirty tracks if the usage status differs from the previous sync,
// if so, we send a new usage with latest status
// if this is our first sync, it will be dirty by default, since we need track usage
dirty = dirty || (quota.Status.Hard == nil || quota.Status.Used == nil)
// Create a usage object that is based on the quota resource version
usage := api.ResourceQuota{
ObjectMeta: api.ObjectMeta{
Name: quota.Name,
Namespace: quota.Namespace,
ResourceVersion: quota.ResourceVersion,
Labels: quota.Labels,
Annotations: quota.Annotations},
Status: api.ResourceQuotaStatus{
Hard: api.ResourceList{},
Used: api.ResourceList{},
},
}
// set the hard values supported on the quota
for k, v := range quota.Spec.Hard {
usage.Status.Hard[k] = *v.Copy()
}
// set any last known observed status values for usage
for k, v := range quota.Status.Used {
usage.Status.Used[k] = *v.Copy()
}
set := map[api.ResourceName]bool{}
for k := range usage.Status.Hard {
set[k] = true
}
pods := &api.PodList{}
if set[api.ResourcePods] || set[api.ResourceMemory] || set[api.ResourceCPU] {
pods, err = rq.kubeClient.Core().Pods(usage.Namespace).List(api.ListOptions{})
if err != nil {
return err
}
}
filteredPods := FilterQuotaPods(pods.Items)
// iterate over each resource, and update observation
for k := range usage.Status.Hard {
// look if there is a used value, if none, we are definitely dirty
prevQuantity, found := usage.Status.Used[k]
if !found {
dirty = true
}
var value *resource.Quantity
switch k {
case api.ResourcePods:
value = resource.NewQuantity(int64(len(filteredPods)), resource.DecimalSI)
case api.ResourceServices:
items, err := rq.kubeClient.Core().Services(usage.Namespace).List(api.ListOptions{})
if err != nil {
return err
}
value = resource.NewQuantity(int64(len(items.Items)), resource.DecimalSI)
case api.ResourceReplicationControllers:
items, err := rq.kubeClient.Core().ReplicationControllers(usage.Namespace).List(api.ListOptions{})
if err != nil {
return err
}
value = resource.NewQuantity(int64(len(items.Items)), resource.DecimalSI)
case api.ResourceQuotas:
items, err := rq.kubeClient.Core().ResourceQuotas(usage.Namespace).List(api.ListOptions{})
if err != nil {
return err
}
value = resource.NewQuantity(int64(len(items.Items)), resource.DecimalSI)
case api.ResourceSecrets:
items, err := rq.kubeClient.Core().Secrets(usage.Namespace).List(api.ListOptions{})
if err != nil {
return err
}
value = resource.NewQuantity(int64(len(items.Items)), resource.DecimalSI)
case api.ResourcePersistentVolumeClaims:
items, err := rq.kubeClient.Core().PersistentVolumeClaims(usage.Namespace).List(api.ListOptions{})
if err != nil {
return err
}
value = resource.NewQuantity(int64(len(items.Items)), resource.DecimalSI)
case api.ResourceMemory:
value = PodsRequests(filteredPods, api.ResourceMemory)
case api.ResourceCPU:
value = PodsRequests(filteredPods, api.ResourceCPU)
}
// ignore fields we do not understand (assume another controller is tracking it)
if value != nil {
// see if the value has changed
dirty = dirty || (value.Value() != prevQuantity.Value())
// just update the value
usage.Status.Used[k] = *value
}
}
// update the usage only if it changed
if dirty {
_, err = rq.kubeClient.Core().ResourceQuotas(usage.Namespace).UpdateStatus(&usage)
return err
}
return nil
}
// limitRequestRatioConstraint enforces the limit to request ratio over the specified resource
func limitRequestRatioConstraint(limitType api.LimitType, resourceName api.ResourceName, enforced resource.Quantity, request api.ResourceList, limit api.ResourceList) error {
req, reqExists := request[resourceName]
lim, limExists := limit[resourceName]
observedReqValue, observedLimValue, enforcedValue := requestLimitEnforcedValues(req, lim, enforced)
if !reqExists || (observedReqValue == int64(0)) {
return fmt.Errorf("%s max limit to request ratio per %s is %s, but no request is specified or request is 0.", resourceName, limitType, enforced.String())
}
if !limExists || (observedLimValue == int64(0)) {
return fmt.Errorf("%s max limit to request ratio per %s is %s, but no limit is specified or limit is 0.", resourceName, limitType, enforced.String())
}
observedValue := observedLimValue / observedReqValue
if observedValue > enforcedValue {
return fmt.Errorf("%s max limit to request ratio per %s is %s, but provided ratio is %d.", resourceName, limitType, enforced.String(), observedValue)
}
return nil
}