// clusterSyncLoop observes running clusters changes, and apply all services to new added cluster // and add dns records for the changes func (s *ServiceController) clusterSyncLoop() { var servicesToUpdate []*cachedService // should we remove cache for cluster from ready to not ready? should remove the condition predicate if no clusters, err := s.clusterStore.ClusterCondition(getClusterConditionPredicate()).List() if err != nil { glog.Infof("Fail to get cluster list") return } newClusters := clustersFromList(&clusters) var newSet, increase sets.String newSet = sets.NewString(newClusters...) if newSet.Equal(s.knownClusterSet) { // The set of cluster names in the services in the federation hasn't changed, but we can retry // updating any services that we failed to update last time around. servicesToUpdate = s.updateDNSRecords(servicesToUpdate, newClusters) return } glog.Infof("Detected change in list of cluster names. New set: %v, Old set: %v", newSet, s.knownClusterSet) increase = newSet.Difference(s.knownClusterSet) // do nothing when cluster is removed. if increase != nil { // Try updating all services, and save the ones that fail to try again next // round. servicesToUpdate = s.serviceCache.allServices() numServices := len(servicesToUpdate) for newCluster := range increase { glog.Infof("New cluster observed %s", newCluster) s.updateAllServicesToCluster(servicesToUpdate, newCluster) } servicesToUpdate = s.updateDNSRecords(servicesToUpdate, newClusters) glog.Infof("Successfully updated %d out of %d DNS records to direct traffic to the updated cluster", numServices-len(servicesToUpdate), numServices) } s.knownClusterSet = newSet }
// enforcePodContainerConstraints checks for required resources that are not set on this container and // adds them to missingSet. func enforcePodContainerConstraints(container *api.Container, requiredSet, missingSet sets.String) { requests := container.Resources.Requests limits := container.Resources.Limits containerUsage := podUsageHelper(requests, limits) containerSet := quota.ToSet(quota.ResourceNames(containerUsage)) if !containerSet.Equal(requiredSet) { difference := requiredSet.Difference(containerSet) missingSet.Insert(difference.List()...) } }
// DialFromContainers executes a curl via kubectl exec in a test container, // which might then translate to a tcp or udp request based on the protocol // argument in the url. // - minTries is the minimum number of curl attempts required before declaring // success. Set to 0 if you'd like to return as soon as all endpoints respond // at least once. // - maxTries is the maximum number of curl attempts. If this many attempts pass // and we don't see all expected endpoints, the test fails. // - expectedEps is the set of endpointnames to wait for. Typically this is also // the hostname reported by each pod in the service through /hostName. // maxTries == minTries will confirm that we see the expected endpoints and no // more for maxTries. Use this if you want to eg: fail a readiness check on a // pod and confirm it doesn't show up as an endpoint. func (config *NetworkingTestConfig) DialFromContainer(protocol, containerIP, targetIP string, containerHttpPort, targetPort, maxTries, minTries int, expectedEps sets.String) { cmd := fmt.Sprintf("curl -q -s 'http://%s:%d/dial?request=hostName&protocol=%s&host=%s&port=%d&tries=1'", containerIP, containerHttpPort, protocol, targetIP, targetPort) eps := sets.NewString() for i := 0; i < maxTries; i++ { stdout, stderr, err := config.f.ExecShellInPodWithFullOutput(config.HostTestContainerPod.Name, cmd) if err != nil { // A failure to kubectl exec counts as a try, not a hard fail. // Also note that we will keep failing for maxTries in tests where // we confirm unreachability. Logf("Failed to execute %q: %v, stdout: %q, stderr %q", cmd, err, stdout, stderr) } else { var output map[string][]string if err := json.Unmarshal([]byte(stdout), &output); err != nil { Logf("WARNING: Failed to unmarshal curl response. Cmd %v run in %v, output: %s, err: %v", cmd, config.HostTestContainerPod.Name, stdout, err) continue } for _, hostName := range output["responses"] { trimmed := strings.TrimSpace(hostName) if trimmed != "" { eps.Insert(trimmed) } } } Logf("Waiting for endpoints: %v", expectedEps.Difference(eps)) // Check against i+1 so we exit if minTries == maxTries. if (eps.Equal(expectedEps) || eps.Len() == 0 && expectedEps.Len() == 0) && i+1 >= minTries { return } // TODO: get rid of this delay #36281 time.Sleep(hitEndpointRetryDelay) } config.diagnoseMissingEndpoints(eps) Failf("Failed to find expected endpoints:\nTries %d\nCommand %v\nretrieved %v\nexpected %v\n", minTries, cmd, eps, expectedEps) }
// DialFromNode executes a tcp or udp request based on protocol via kubectl exec // in a test container running with host networking. // - minTries is the minimum number of curl attempts required before declaring // success. Set to 0 if you'd like to return as soon as all endpoints respond // at least once. // - maxTries is the maximum number of curl attempts. If this many attempts pass // and we don't see all expected endpoints, the test fails. // maxTries == minTries will confirm that we see the expected endpoints and no // more for maxTries. Use this if you want to eg: fail a readiness check on a // pod and confirm it doesn't show up as an endpoint. func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targetPort, maxTries, minTries int, expectedEps sets.String) { var cmd string if protocol == "udp" { // TODO: It would be enough to pass 1s+epsilon to timeout, but unfortunately // busybox timeout doesn't support non-integer values. cmd = fmt.Sprintf("echo 'hostName' | timeout -t 2 nc -w 1 -u %s %d", targetIP, targetPort) } else { cmd = fmt.Sprintf("timeout -t 15 curl -q -s --connect-timeout 1 http://%s:%d/hostName", targetIP, targetPort) } // TODO: This simply tells us that we can reach the endpoints. Check that // the probability of hitting a specific endpoint is roughly the same as // hitting any other. eps := sets.NewString() filterCmd := fmt.Sprintf("%s | grep -v '^\\s*$'", cmd) for i := 0; i < maxTries; i++ { stdout, stderr, err := config.f.ExecShellInPodWithFullOutput(config.HostTestContainerPod.Name, filterCmd) if err != nil || len(stderr) > 0 { // A failure to exec command counts as a try, not a hard fail. // Also note that we will keep failing for maxTries in tests where // we confirm unreachability. Logf("Failed to execute %q: %v, stdout: %q, stderr: %q", filterCmd, err, stdout, stderr) } else { trimmed := strings.TrimSpace(stdout) if trimmed != "" { eps.Insert(trimmed) } } Logf("Waiting for %+v endpoints, got endpoints %+v", expectedEps.Difference(eps), eps) // Check against i+1 so we exit if minTries == maxTries. if (eps.Equal(expectedEps) || eps.Len() == 0 && expectedEps.Len() == 0) && i+1 >= minTries { return } // TODO: get rid of this delay #36281 time.Sleep(hitEndpointRetryDelay) } config.diagnoseMissingEndpoints(eps) Failf("Failed to find expected endpoints:\nTries %d\nCommand %v\nretrieved %v\nexpected %v\n", minTries, cmd, eps, expectedEps) }
func TestHitNodesFromOutsideWithCount(externalIP string, httpPort int32, timeout time.Duration, expectedHosts sets.String, countToSucceed int) error { Logf("Waiting up to %v for satisfying expectedHosts for %v times", timeout, countToSucceed) hittedHosts := sets.NewString() count := 0 condition := func() (bool, error) { var respBody bytes.Buffer reached, err := TestReachableHTTPWithContentTimeout(externalIP, int(httpPort), "/hostname", "", &respBody, 1*time.Second) if err != nil || !reached { return false, nil } hittedHost := strings.TrimSpace(respBody.String()) if !expectedHosts.Has(hittedHost) { Logf("Error hitting unexpected host: %v, reset counter: %v", hittedHost, count) count = 0 return false, nil } if !hittedHosts.Has(hittedHost) { hittedHosts.Insert(hittedHost) Logf("Missing %+v, got %+v", expectedHosts.Difference(hittedHosts), hittedHosts) } if hittedHosts.Equal(expectedHosts) { count++ if count >= countToSucceed { return true, nil } } return false, nil } if err := wait.Poll(time.Second, timeout, condition); err != nil { return fmt.Errorf("error waiting for expectedHosts: %v, hittedHosts: %v, count: %v, expected count: %v", expectedHosts, hittedHosts, count, countToSucceed) } return nil }