func (t *tcShaper) nextClassID() (int, error) {
data, err := t.e.Command("tc", "class", "show", "dev", t.iface).CombinedOutput()
if err != nil {
return -1, err
}
scanner := bufio.NewScanner(bytes.NewBuffer(data))
classes := sets.String{}
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// skip empty lines
if len(line) == 0 {
continue
}
parts := strings.Split(line, " ")
// expected tc line:
// class htb 1:1 root prio 0 rate 1000Kbit ceil 1000Kbit burst 1600b cburst 1600b
if len(parts) != 14 {
return -1, fmt.Errorf("unexpected output from tc: %s (%v)", scanner.Text(), parts)
}
classes.Insert(parts[2])
}
// Make sure it doesn't go forever
for nextClass := 1; nextClass < 10000; nextClass++ {
if !classes.Has(fmt.Sprintf("1:%d", nextClass)) {
return nextClass, nil
}
}
// This should really never happen
return -1, fmt.Errorf("exhausted class space, please try again")
}
// Print formats and prints the give PodDiff.
func (p PodDiff) String(ignorePhases sets.String) string {
ret := ""
for name, info := range p {
if ignorePhases.Has(info.phase) {
continue
}
if info.phase == nonExist {
ret += fmt.Sprintf("Pod %v was deleted, had phase %v and host %v\n", name, info.oldPhase, info.oldHostname)
continue
}
phaseChange, hostChange := false, false
msg := fmt.Sprintf("Pod %v ", name)
if info.oldPhase != info.phase {
phaseChange = true
if info.oldPhase == nonExist {
msg += fmt.Sprintf("in phase %v ", info.phase)
} else {
msg += fmt.Sprintf("went from phase: %v -> %v ", info.oldPhase, info.phase)
}
}
if info.oldHostname != info.hostname {
hostChange = true
if info.oldHostname == nonExist || info.oldHostname == "" {
msg += fmt.Sprintf("assigned host %v ", info.hostname)
} else {
msg += fmt.Sprintf("went from host: %v -> %v ", info.oldHostname, info.hostname)
}
}
if phaseChange || hostChange {
ret += msg + "\n"
}
}
return ret
}
func findKnownValue(parts []string, valueOptions sets.String) int {
for i := range parts {
if valueOptions.Has(parts[i]) {
return i
}
}
return -1
}
// filterUnmountedVolumes adds each element of expectedVolumes that is not in
// mountedVolumes to a list of unmountedVolumes and returns it.
func filterUnmountedVolumes(
mountedVolumes sets.String, expectedVolumes []string) []string {
unmountedVolumes := []string{}
for _, expectedVolume := range expectedVolumes {
if !mountedVolumes.Has(expectedVolume) {
unmountedVolumes = append(unmountedVolumes, expectedVolume)
}
}
return unmountedVolumes
}
// PodMatchesTermsNamespaceAndSelector returns true if the given <pod>
// matches the namespace and selector defined by <affinityPod>`s <term>.
func PodMatchesTermsNamespaceAndSelector(pod *v1.Pod, namespaces sets.String, selector labels.Selector) bool {
if len(namespaces) != 0 && !namespaces.Has(pod.Namespace) {
return false
}
if !selector.Matches(labels.Set(pod.Labels)) {
return false
}
return true
}
// BasicLongRunningRequestCheck returns true if the given request has one of the specified verbs or one of the specified subresources
func BasicLongRunningRequestCheck(longRunningVerbs, longRunningSubresources sets.String) LongRunningRequestCheck {
return func(r *http.Request, requestInfo *apirequest.RequestInfo) bool {
if longRunningVerbs.Has(requestInfo.Verb) {
return true
}
if requestInfo.IsResourceRequest && longRunningSubresources.Has(requestInfo.Subresource) {
return true
}
return false
}
}
// diagnoseMissingEndpoints prints debug information about the endpoints that
// are NOT in the given list of foundEndpoints. These are the endpoints we
// expected a response from.
func (config *NetworkingTestConfig) diagnoseMissingEndpoints(foundEndpoints sets.String) {
for _, e := range config.EndpointPods {
if foundEndpoints.Has(e.Name) {
continue
}
Logf("\nOutput of kubectl describe pod %v/%v:\n", e.Namespace, e.Name)
desc, _ := RunKubectl(
"describe", "pod", e.Name, fmt.Sprintf("--namespace=%v", e.Namespace))
Logf(desc)
}
}
// Triggers the application of udev rules by calling "udevadm trigger
// --action=change" for newly created "/dev/sd*" drives (exist only in
// after set). This is workaround for Issue #7972. Once the underlying
// issue has been resolved, this may be removed.
func udevadmChangeToNewDrives(sdBeforeSet sets.String) error {
sdAfter, err := filepath.Glob(diskSDPattern)
if err != nil {
return fmt.Errorf("Error filepath.Glob(\"%s\"): %v\r\n", diskSDPattern, err)
}
for _, sd := range sdAfter {
if !sdBeforeSet.Has(sd) {
return udevadmChangeToDrive(sd)
}
}
return nil
}
// cleanupBandwidthLimits updates the status of bandwidth-limited containers
// and ensures that only the appropriate CIDRs are active on the node.
func (kl *Kubelet) cleanupBandwidthLimits(allPods []*v1.Pod) error {
if kl.shaper == nil {
return nil
}
currentCIDRs, err := kl.shaper.GetCIDRs()
if err != nil {
return err
}
possibleCIDRs := sets.String{}
for ix := range allPods {
pod := allPods[ix]
ingress, egress, err := bandwidth.ExtractPodBandwidthResources(pod.Annotations)
if err != nil {
return err
}
if ingress == nil && egress == nil {
glog.V(8).Infof("Not a bandwidth limited container...")
continue
}
status, found := kl.statusManager.GetPodStatus(pod.UID)
if !found {
// TODO(random-liu): Cleanup status get functions. (issue #20477)
s, err := kl.containerRuntime.GetPodStatus(pod.UID, pod.Name, pod.Namespace)
if err != nil {
return err
}
status = kl.generateAPIPodStatus(pod, s)
}
if status.Phase == v1.PodRunning {
possibleCIDRs.Insert(fmt.Sprintf("%s/32", status.PodIP))
}
}
for _, cidr := range currentCIDRs {
if !possibleCIDRs.Has(cidr) {
glog.V(2).Infof("Removing CIDR: %s (%v)", cidr, possibleCIDRs)
if err := kl.shaper.Reset(cidr); err != nil {
return err
}
}
}
return nil
}
func AccumulateUniqueHostPorts(containers []v1.Container, accumulator *sets.String, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
for ci, ctr := range containers {
idxPath := fldPath.Index(ci)
portsPath := idxPath.Child("ports")
for pi := range ctr.Ports {
idxPath := portsPath.Index(pi)
port := ctr.Ports[pi].HostPort
if port == 0 {
continue
}
str := fmt.Sprintf("%d/%s", port, ctr.Ports[pi].Protocol)
if accumulator.Has(str) {
allErrs = append(allErrs, field.Duplicate(idxPath.Child("hostPort"), str))
} else {
accumulator.Insert(str)
}
}
}
return allErrs
}
// NegotiateVersion queries the server's supported api versions to find
// a version that both client and server support.
// - If no version is provided, try registered client versions in order of
// preference.
// - If version is provided and the server does not support it,
// return an error.
func NegotiateVersion(client DiscoveryInterface, requiredGV *schema.GroupVersion, clientRegisteredGVs []schema.GroupVersion) (*schema.GroupVersion, error) {
clientVersions := sets.String{}
for _, gv := range clientRegisteredGVs {
clientVersions.Insert(gv.String())
}
groups, err := client.ServerGroups()
if err != nil {
// This is almost always a connection error, and higher level code should treat this as a generic error,
// not a negotiation specific error.
return nil, err
}
versions := metav1.ExtractGroupVersions(groups)
serverVersions := sets.String{}
for _, v := range versions {
serverVersions.Insert(v)
}
// If version explicitly requested verify that both client and server support it.
// If server does not support warn, but try to negotiate a lower version.
if requiredGV != nil {
if !clientVersions.Has(requiredGV.String()) {
return nil, fmt.Errorf("client does not support API version %q; client supported API versions: %v", requiredGV, clientVersions)
}
// If the server supports no versions, then we should just use the preferredGV
// This can happen because discovery fails due to 403 Forbidden errors
if len(serverVersions) == 0 {
return requiredGV, nil
}
if serverVersions.Has(requiredGV.String()) {
return requiredGV, nil
}
// If we are using an explicit config version the server does not support, fail.
return nil, fmt.Errorf("server does not support API version %q", requiredGV)
}
for _, clientGV := range clientRegisteredGVs {
if serverVersions.Has(clientGV.String()) {
// Version was not explicitly requested in command config (--api-version).
// Ok to fall back to a supported version with a warning.
// TODO: caesarxuchao: enable the warning message when we have
// proper fix. Please refer to issue #14895.
// if len(version) != 0 {
// glog.Warningf("Server does not support API version '%s'. Falling back to '%s'.", version, clientVersion)
// }
t := clientGV
return &t, nil
}
}
// if we have no server versions and we have no required version, choose the first clientRegisteredVersion
if len(serverVersions) == 0 && len(clientRegisteredGVs) > 0 {
return &clientRegisteredGVs[0], nil
}
return nil, fmt.Errorf("failed to negotiate an api version; server supports: %v, client supports: %v",
serverVersions, clientVersions)
}
func filterInvalidPods(pods []*v1.Pod, source string, recorder record.EventRecorder) (filtered []*v1.Pod) {
names := sets.String{}
for i, pod := range pods {
var errlist field.ErrorList
// TODO: remove the conversion when validation is performed on versioned objects.
internalPod := &api.Pod{}
if err := v1.Convert_v1_Pod_To_api_Pod(pod, internalPod, nil); err != nil {
name := kubecontainer.GetPodFullName(pod)
glog.Warningf("Pod[%d] (%s) from %s failed to convert to v1, ignoring: %v", i+1, name, source, err)
recorder.Eventf(pod, v1.EventTypeWarning, "FailedConversion", "Error converting pod %s from %s, ignoring: %v", name, source, err)
continue
}
if errs := validation.ValidatePod(internalPod); len(errs) != 0 {
errlist = append(errlist, errs...)
// If validation fails, don't trust it any further -
// even Name could be bad.
} else {
name := kubecontainer.GetPodFullName(pod)
if names.Has(name) {
// TODO: when validation becomes versioned, this gets a bit
// more complicated.
errlist = append(errlist, field.Duplicate(field.NewPath("metadata", "name"), pod.Name))
} else {
names.Insert(name)
}
}
if len(errlist) > 0 {
name := bestPodIdentString(pod)
err := errlist.ToAggregate()
glog.Warningf("Pod[%d] (%s) from %s failed validation, ignoring: %v", i+1, name, source, err)
recorder.Eventf(pod, v1.EventTypeWarning, events.FailedValidation, "Error validating pod %s from %s, ignoring: %v", name, source, err)
continue
}
filtered = append(filtered, pod)
}
return
}
func TestHitNodesFromOutsideWithCount(externalIP string, httpPort int32, timeout time.Duration, expectedHosts sets.String,
countToSucceed int) error {
Logf("Waiting up to %v for satisfying expectedHosts for %v times", timeout, countToSucceed)
hittedHosts := sets.NewString()
count := 0
condition := func() (bool, error) {
var respBody bytes.Buffer
reached, err := TestReachableHTTPWithContentTimeout(externalIP, int(httpPort), "/hostname", "", &respBody,
1*time.Second)
if err != nil || !reached {
return false, nil
}
hittedHost := strings.TrimSpace(respBody.String())
if !expectedHosts.Has(hittedHost) {
Logf("Error hitting unexpected host: %v, reset counter: %v", hittedHost, count)
count = 0
return false, nil
}
if !hittedHosts.Has(hittedHost) {
hittedHosts.Insert(hittedHost)
Logf("Missing %+v, got %+v", expectedHosts.Difference(hittedHosts), hittedHosts)
}
if hittedHosts.Equal(expectedHosts) {
count++
if count >= countToSucceed {
return true, nil
}
}
return false, nil
}
if err := wait.Poll(time.Second, timeout, condition); err != nil {
return fmt.Errorf("error waiting for expectedHosts: %v, hittedHosts: %v, count: %v, expected count: %v",
expectedHosts, hittedHosts, count, countToSucceed)
}
return nil
}
func IsControllerEnabled(name string, disabledByDefaultControllers sets.String, controllers ...string) bool {
hasStar := false
for _, controller := range controllers {
if controller == name {
return true
}
if controller == "-"+name {
return false
}
if controller == "*" {
hasStar = true
}
}
// if we get here, there was no explicit choice
if !hasStar {
// nothing on by default
return false
}
if disabledByDefaultControllers.Has(name) {
return false
}
return true
}
// Instantiates a DefaultRESTMapper based on types registered in the given scheme.
func NewDefaultRESTMapperFromScheme(defaultGroupVersions []schema.GroupVersion, interfacesFunc meta.VersionInterfacesFunc,
importPathPrefix string, ignoredKinds, rootScoped sets.String, scheme *runtime.Scheme) *meta.DefaultRESTMapper {
mapper := meta.NewDefaultRESTMapper(defaultGroupVersions, interfacesFunc)
// enumerate all supported versions, get the kinds, and register with the mapper how to address
// our resources.
for _, gv := range defaultGroupVersions {
for kind, oType := range scheme.KnownTypes(gv) {
gvk := gv.WithKind(kind)
// TODO: Remove import path check.
// We check the import path because we currently stuff both "api" and "extensions" objects
// into the same group within Scheme since Scheme has no notion of groups yet.
if !strings.Contains(oType.PkgPath(), importPathPrefix) || ignoredKinds.Has(kind) {
continue
}
scope := meta.RESTScopeNamespace
if rootScoped.Has(kind) {
scope = meta.RESTScopeRoot
}
mapper.Add(gvk, scope)
}
}
return mapper
}
func ValidateThirdPartyResource(obj *extensions.ThirdPartyResource) field.ErrorList {
allErrs := field.ErrorList{}
allErrs = append(allErrs, apivalidation.ValidateObjectMeta(&obj.ObjectMeta, false, ValidateThirdPartyResourceName, field.NewPath("metadata"))...)
versions := sets.String{}
if len(obj.Versions) == 0 {
allErrs = append(allErrs, field.Required(field.NewPath("versions"), "must specify at least one version"))
}
for ix := range obj.Versions {
version := &obj.Versions[ix]
if len(version.Name) == 0 {
allErrs = append(allErrs, field.Invalid(field.NewPath("versions").Index(ix).Child("name"), version, "must not be empty"))
} else {
for _, msg := range validation.IsDNS1123Label(version.Name) {
allErrs = append(allErrs, field.Invalid(field.NewPath("versions").Index(ix).Child("name"), version, msg))
}
}
if versions.Has(version.Name) {
allErrs = append(allErrs, field.Duplicate(field.NewPath("versions").Index(ix).Child("name"), version))
}
versions.Insert(version.Name)
}
return allErrs
}
nodeList = &v1.NodeList{}
framework.WaitForAllNodesHealthy(cs, time.Minute)
masterNodes, nodeList = framework.GetMasterAndWorkerNodesOrDie(cs)
err := framework.CheckTestingNSDeletedExcept(cs, ns)
framework.ExpectNoError(err)
// Every test case in this suite assumes that cluster add-on pods stay stable and
// cannot be run in parallel with any other test that touches Nodes or Pods.
// It is so because we need to have precise control on what's running in the cluster.
systemPods, err := framework.GetPodsInNamespace(cs, ns, ignoreLabels)
Expect(err).NotTo(HaveOccurred())
systemPodsNo = 0
for _, pod := range systemPods {
if !masterNodes.Has(pod.Spec.NodeName) && pod.DeletionTimestamp == nil {
systemPodsNo++
}
}
err = framework.WaitForPodsRunningReady(cs, api.NamespaceSystem, int32(systemPodsNo), framework.PodReadyBeforeTimeout, ignoreLabels, true)
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeList.Items {
framework.Logf("\nLogging pods the kubelet thinks is on node %v before test", node.Name)
framework.PrintAllKubeletPods(cs, node.Name)
}
})
// This test verifies that max-pods flag works as advertised. It assumes that cluster add-on pods stay stable
func TestSyncAPIs(t *testing.T) {
resourcesNamed := func(names ...string) []expapi.ThirdPartyResource {
result := []expapi.ThirdPartyResource{}
for _, name := range names {
result = append(result, expapi.ThirdPartyResource{ObjectMeta: metav1.ObjectMeta{Name: name}})
}
return result
}
tests := []struct {
list *expapi.ThirdPartyResourceList
apis []string
expectedInstalled []string
expectedRemoved []string
name string
}{
{
list: &expapi.ThirdPartyResourceList{
Items: resourcesNamed("foo.example.com"),
},
expectedInstalled: []string{"foo.example.com"},
name: "simple add",
},
{
list: &expapi.ThirdPartyResourceList{
Items: resourcesNamed("foo.example.com"),
},
apis: []string{
"/apis/example.com",
"/apis/example.com/v1",
},
name: "does nothing",
},
{
list: &expapi.ThirdPartyResourceList{
Items: resourcesNamed("foo.example.com"),
},
apis: []string{
"/apis/example.com",
"/apis/example.com/v1",
"/apis/example.co",
"/apis/example.co/v1",
},
name: "deletes substring API",
expectedRemoved: []string{
"/apis/example.co",
"/apis/example.co/v1",
},
},
{
list: &expapi.ThirdPartyResourceList{
Items: resourcesNamed("foo.example.com", "foo.company.com"),
},
apis: []string{
"/apis/company.com",
"/apis/company.com/v1",
},
expectedInstalled: []string{"foo.example.com"},
name: "adds with existing",
},
{
list: &expapi.ThirdPartyResourceList{
Items: resourcesNamed("foo.example.com"),
},
apis: []string{
"/apis/company.com",
"/apis/company.com/v1",
},
expectedInstalled: []string{"foo.example.com"},
expectedRemoved: []string{"/apis/company.com", "/apis/company.com/v1"},
name: "removes with existing",
},
}
for _, test := range tests {
fake := FakeAPIInterface{
apis: test.apis,
t: t,
}
cntrl := ThirdPartyController{master: &fake}
if err := cntrl.syncResourceList(test.list); err != nil {
t.Errorf("[%s] unexpected error: %v", test.name, err)
}
if len(test.expectedInstalled) != len(fake.installed) {
t.Errorf("[%s] unexpected installed APIs: %d, expected %d (%#v)", test.name, len(fake.installed), len(test.expectedInstalled), fake.installed[0])
continue
} else {
names := sets.String{}
for ix := range fake.installed {
names.Insert(fake.installed[ix].Name)
}
for _, name := range test.expectedInstalled {
if !names.Has(name) {
t.Errorf("[%s] missing installed API: %s", test.name, name)
}
}
}
if len(test.expectedRemoved) != len(fake.removed) {
t.Errorf("[%s] unexpected installed APIs: %d, expected %d", test.name, len(fake.removed), len(test.expectedRemoved))
continue
} else {
names := sets.String{}
names.Insert(fake.removed...)
for _, name := range test.expectedRemoved {
if !names.Has(name) {
t.Errorf("[%s] missing removed API: %s (%s)", test.name, name, names)
}
}
}
}
}
// ipamGarbageCollection will release unused IP.
// kubenet uses the CNI bridge plugin, which stores allocated ips on file. Each
// file created under defaultIPAMDir has the format: ip/container-hash. So this
// routine looks for hashes that are not reported by the currently running docker,
// and invokes DelNetwork on each one. Note that this will only work for the
// current CNI bridge plugin, because we have no way of finding the NetNs.
func (plugin *kubenetNetworkPlugin) ipamGarbageCollection() {
glog.V(2).Infof("Starting IP garbage collection")
ipamDir := filepath.Join(defaultIPAMDir, KubenetPluginName)
files, err := ioutil.ReadDir(ipamDir)
if err != nil {
glog.Errorf("Failed to list files in %q: %v", ipamDir, err)
return
}
// gather containerIDs for allocated ips
ipContainerIdMap := make(map[string]string)
for _, file := range files {
// skip non checkpoint file
if ip := net.ParseIP(file.Name()); ip == nil {
continue
}
content, err := ioutil.ReadFile(filepath.Join(ipamDir, file.Name()))
if err != nil {
glog.Errorf("Failed to read file %v: %v", file, err)
}
ipContainerIdMap[file.Name()] = strings.TrimSpace(string(content))
}
// gather infra container IDs of current running Pods
runningContainerIDs := utilsets.String{}
pods, err := plugin.getNonExitedPods()
if err != nil {
glog.Errorf("Failed to get pods: %v", err)
return
}
for _, pod := range pods {
containerID, err := plugin.host.GetRuntime().GetPodContainerID(pod)
if err != nil {
glog.Warningf("Failed to get infra containerID of %q/%q: %v", pod.Namespace, pod.Name, err)
continue
}
runningContainerIDs.Insert(strings.TrimSpace(containerID.ID))
}
// release leaked ips
for ip, containerID := range ipContainerIdMap {
// if the container is not running, release IP
if runningContainerIDs.Has(containerID) {
continue
}
// CNI requires all config to be presented, although only containerID is needed in this case
rt := &libcni.RuntimeConf{
ContainerID: containerID,
IfName: network.DefaultInterfaceName,
// TODO: How do we find the NetNs of an exited container? docker inspect
// doesn't show us the pid, so we probably need to checkpoint
NetNS: "",
}
glog.V(2).Infof("Releasing IP %q allocated to %q.", ip, containerID)
// CNI bridge plugin should try to release IP and then return
if err := plugin.cniConfig.DelNetwork(plugin.netConfig, rt); err != nil {
glog.Errorf("Error while releasing IP: %v", err)
}
}
}