func verifyExpectedRcsExistAndGetExpectedPods(c *client.Client) ([]string, error) {
expectedPods := []string{}
// Iterate over the labels that identify the replication controllers that we
// want to check. The rcLabels contains the value values for the k8s-app key
// that identify the replication controllers that we want to check. Using a label
// rather than an explicit name is preferred because the names will typically have
// a version suffix e.g. heapster-monitoring-v1 and this will change after a rolling
// update e.g. to heapster-monitoring-v2. By using a label query we can check for the
// situaiton when a heapster-monitoring-v1 and heapster-monitoring-v2 replication controller
// is running (which would be an error except during a rolling update).
for _, rcLabel := range rcLabels {
rcList, err := c.ReplicationControllers(api.NamespaceSystem).List(labels.Set{"k8s-app": rcLabel}.AsSelector())
if err != nil {
return nil, err
}
if len(rcList.Items) != 1 {
return nil, fmt.Errorf("expected to find one replica for RC with label %s but got %d",
rcLabel, len(rcList.Items))
}
for _, rc := range rcList.Items {
podList, err := c.Pods(api.NamespaceSystem).List(labels.Set(rc.Spec.Selector).AsSelector(), fields.Everything())
if err != nil {
return nil, err
}
for _, pod := range podList.Items {
expectedPods = append(expectedPods, string(pod.UID))
}
}
}
return expectedPods, nil
}
func validateRouter(c *k8sclient.Client, f *cmdutil.Factory) (Result, error) {
ns, _, err := f.DefaultNamespace()
if err != nil {
return Failure, err
}
requirement, err := labels.NewRequirement("router", labels.EqualsOperator, kutil.NewStringSet("router"))
if err != nil {
return Failure, err
}
label := labels.LabelSelector{*requirement}
rc, err := c.ReplicationControllers(ns).List(label)
if err != nil {
util.Fatalf("Failed to get PersistentVolumeClaims, %s in namespace %s\n", err, ns)
}
if rc != nil {
items := rc.Items
if len(items) > 0 {
return Success, err
}
}
//util.Fatalf("No router running in namespace %s\n", ns)
// TODO lets create a router
return Failure, err
}
func NewTurboScheduler(kubeClient *client.Client, vmturboMeta *vmtmeta.VMTMeta) *TurboScheduler {
scheduledPodLister := &cache.StoreToPodLister{}
podQueue := cache.NewFIFO(cache.MetaNamespaceKeyFunc)
modeler := scheduler.NewSimpleModeler(&cache.StoreToPodLister{Store: podQueue}, scheduledPodLister)
bindPodsQPS := float32(15.0)
bindPodsBurst := 20
rateLimiter := util.NewTokenBucketRateLimiter(bindPodsQPS, bindPodsBurst)
config := &Config{
Modeler: modeler,
Binder: &binder{kubeClient},
BindPodsRateLimiter: rateLimiter,
}
eventBroadcaster := record.NewBroadcaster()
config.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "turboscheduler"})
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
vmtSched := vmtscheduler.NewVMTScheduler(kubeClient, vmturboMeta)
glog.V(3).Infof("VMTScheduler is set: %++v", vmtSched)
defaultSched := defaultscheduler.NewDefaultScheduler(kubeClient)
glog.V(3).Infof("DefaultScheduler is set: %++v", defaultSched)
return &TurboScheduler{
config: config,
vmtScheduler: vmtSched,
defaultScheduler: defaultSched,
}
}
// StartPods check for numPods in TestNS. If they exist, it no-ops, otherwise it starts up
// a temp rc, scales it to match numPods, then deletes the rc leaving behind the pods.
func StartPods(numPods int, host string, restClient *client.Client) error {
start := time.Now()
defer func() {
glog.Infof("StartPods took %v with numPods %d", time.Since(start), numPods)
}()
hostField := fields.OneTermEqualSelector(client.PodHost, host)
pods, err := restClient.Pods(TestNS).List(labels.Everything(), hostField)
if err != nil || len(pods.Items) == numPods {
return err
}
glog.Infof("Found %d pods that match host %v, require %d", len(pods.Items), hostField, numPods)
// For the sake of simplicity, assume all pods in TestNS have selectors matching TestRCManifest.
controller := RCFromManifest(TestRCManifest)
// Make the rc unique to the given host.
controller.Spec.Replicas = numPods
controller.Spec.Template.Spec.NodeName = host
controller.Name = controller.Name + host
controller.Spec.Selector["host"] = host
controller.Spec.Template.Labels["host"] = host
if rc, err := StartRC(controller, restClient); err != nil {
return err
} else {
// Delete the rc, otherwise when we restart master components for the next benchmark
// the rc controller will race with the pods controller in the rc manager.
return restClient.ReplicationControllers(TestNS).Delete(rc.Name)
}
}
// testHostIP tests that a pod gets a host IP
func testHostIP(c *client.Client, ns string, pod *api.Pod) {
podClient := c.Pods(ns)
By("creating pod")
defer podClient.Delete(pod.Name, nil)
_, err := podClient.Create(pod)
if err != nil {
Failf("Failed to create pod: %v", err)
}
By("ensuring that pod is running and has a hostIP")
// Wait for the pods to enter the running state. Waiting loops until the pods
// are running so non-running pods cause a timeout for this test.
err = waitForPodRunningInNamespace(c, pod.Name, ns)
Expect(err).NotTo(HaveOccurred())
// Try to make sure we get a hostIP for each pod.
hostIPTimeout := 2 * time.Minute
t := time.Now()
for {
p, err := podClient.Get(pod.Name)
Expect(err).NotTo(HaveOccurred())
if p.Status.HostIP != "" {
Logf("Pod %s has hostIP: %s", p.Name, p.Status.HostIP)
break
}
if time.Since(t) >= hostIPTimeout {
Failf("Gave up waiting for hostIP of pod %s after %v seconds",
p.Name, time.Since(t).Seconds())
}
Logf("Retrying to get the hostIP of pod %s", p.Name)
time.Sleep(5 * time.Second)
}
}
func createRoutesForDomain(ns string, domain string, c *k8sclient.Client, oc *oclient.Client, fac *cmdutil.Factory) error {
rc, err := c.Services(ns).List(labels.Everything())
if err != nil {
util.Errorf("Failed to load services in namespace %s with error %v", ns, err)
return err
}
items := rc.Items
for _, service := range items {
// TODO use the external load balancer as a way to know if we should create a route?
name := service.ObjectMeta.Name
if name != "kubernetes" {
routes := oc.Routes(ns)
_, err = routes.Get(name)
if err != nil {
hostName := name + "." + domain
route := rapi.Route{
ObjectMeta: kapi.ObjectMeta{
Name: name,
},
Host: hostName,
ServiceName: name,
}
// lets create the route
_, err = routes.Create(&route)
if err != nil {
util.Errorf("Failed to create the route %s with error %v", name, err)
return err
}
}
}
}
return nil
}
func verifyResult(c *client.Client, podName string, ns string, oldNotRunning int) {
allPods, err := c.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything())
expectNoError(err)
_, notRunningPods := getPodsNumbers(allPods)
schedEvents, err := c.Events(ns).List(
labels.Everything(),
fields.Set{
"involvedObject.kind": "Pod",
"involvedObject.name": podName,
"involvedObject.namespace": ns,
"source": "scheduler",
"reason": "FailedScheduling",
}.AsSelector())
expectNoError(err)
printed := false
printOnce := func(msg string) string {
if !printed {
printed = true
return msg
} else {
return ""
}
}
Expect(notRunningPods).To(Equal(1+oldNotRunning), printOnce(fmt.Sprintf("Pods found in the cluster: %#v", allPods)))
Expect(schedEvents.Items).ToNot(BeEmpty(), printOnce(fmt.Sprintf("Pods found in the cluster: %#v", allPods)))
}
func CheckCadvisorHealthOnAllNodes(c *client.Client, timeout time.Duration) {
By("getting list of nodes")
nodeList, err := c.Nodes().List(labels.Everything(), fields.Everything())
expectNoError(err)
var errors []error
retries := maxRetries
for {
errors = []error{}
for _, node := range nodeList.Items {
// cadvisor is not accessible directly unless its port (4194 by default) is exposed.
// Here, we access '/stats/' REST endpoint on the kubelet which polls cadvisor internally.
statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
By(fmt.Sprintf("Querying stats from node %s using url %s", node.Name, statsResource))
_, err = c.Get().AbsPath(statsResource).Timeout(timeout).Do().Raw()
if err != nil {
errors = append(errors, err)
}
}
if len(errors) == 0 {
return
}
if retries--; retries <= 0 {
break
}
Logf("failed to retrieve kubelet stats -\n %v", errors)
time.Sleep(sleepDuration)
}
Failf("Failed after retrying %d times for cadvisor to be healthy on all nodes. Errors:\n%v", maxRetries, errors)
}
func CreateNewControllerFromCurrentController(c *client.Client, namespace, oldName, newName, image, deploymentKey string) (*api.ReplicationController, error) {
// load the old RC into the "new" RC
newRc, err := c.ReplicationControllers(namespace).Get(oldName)
if err != nil {
return nil, err
}
if len(newRc.Spec.Template.Spec.Containers) > 1 {
// TODO: support multi-container image update.
return nil, goerrors.New("Image update is not supported for multi-container pods")
}
if len(newRc.Spec.Template.Spec.Containers) == 0 {
return nil, goerrors.New(fmt.Sprintf("Pod has no containers! (%v)", newRc))
}
newRc.Spec.Template.Spec.Containers[0].Image = image
newHash, err := api.HashObject(newRc, c.Codec)
if err != nil {
return nil, err
}
if len(newName) == 0 {
newName = fmt.Sprintf("%s-%s", newRc.Name, newHash)
}
newRc.Name = newName
newRc.Spec.Selector[deploymentKey] = newHash
newRc.Spec.Template.Labels[deploymentKey] = newHash
// Clear resource version after hashing so that identical updates get different hashes.
newRc.ResourceVersion = ""
return newRc, nil
}
func runMasterServiceTest(client *client.Client) {
time.Sleep(12 * time.Second)
svcList, err := client.Services(api.NamespaceDefault).List(labels.Everything())
if err != nil {
glog.Fatalf("unexpected error listing services: %v", err)
}
var foundRW bool
found := util.StringSet{}
for i := range svcList.Items {
found.Insert(svcList.Items[i].Name)
if svcList.Items[i].Name == "kubernetes" {
foundRW = true
}
}
if foundRW {
ep, err := client.Endpoints(api.NamespaceDefault).Get("kubernetes")
if err != nil {
glog.Fatalf("unexpected error listing endpoints for kubernetes service: %v", err)
}
if countEndpoints(ep) == 0 {
glog.Fatalf("no endpoints for kubernetes service: %v", ep)
}
} else {
glog.Errorf("no RW service found: %v", found)
glog.Fatal("Kubernetes service test failed")
}
glog.Infof("Master service test passed.")
}
// Creates a replication controller that serves its hostname and a service on top of it.
func startServeHostnameService(c *client.Client, ns, name string, port, replicas int) ([]string, string, error) {
podNames := make([]string, replicas)
_, err := c.Services(ns).Create(&api.Service{
ObjectMeta: api.ObjectMeta{
Name: name,
},
Spec: api.ServiceSpec{
Ports: []api.ServicePort{{
Port: port,
TargetPort: util.NewIntOrStringFromInt(9376),
Protocol: "TCP",
}},
Selector: map[string]string{
"name": name,
},
},
})
if err != nil {
return podNames, "", err
}
var createdPods []*api.Pod
maxContainerFailures := 0
config := RCConfig{
Client: c,
Image: "gcr.io/google_containers/serve_hostname:1.1",
Name: name,
Namespace: ns,
PollInterval: 3 * time.Second,
Timeout: 30 * time.Second,
Replicas: replicas,
CreatedPods: &createdPods,
MaxContainerFailures: &maxContainerFailures,
}
err = RunRC(config)
if err != nil {
return podNames, "", err
}
if len(createdPods) != replicas {
return podNames, "", fmt.Errorf("Incorrect number of running pods: %v", len(createdPods))
}
for i := range createdPods {
podNames[i] = createdPods[i].ObjectMeta.Name
}
sort.StringSlice(podNames).Sort()
service, err := c.Services(ns).Get(name)
if err != nil {
return podNames, "", err
}
if service.Spec.ClusterIP == "" {
return podNames, "", fmt.Errorf("Service IP is blank for %v", name)
}
serviceIP := service.Spec.ClusterIP
return podNames, serviceIP, nil
}
// Performs a get on a node proxy endpoint given the nodename and rest client.
func nodeProxyRequest(c *client.Client, node, endpoint string) client.Result {
return c.Get().
Prefix("proxy").
Resource("nodes").
Name(fmt.Sprintf("%v:%v", node, ports.KubeletPort)).
Suffix(endpoint).
Do()
}
func newProjectAuthorizationCache(openshiftClient *osclient.Client, kubeClient *kclient.Client,
policyClient policyclient.ReadOnlyPolicyClient) *projectauth.AuthorizationCache {
return projectauth.NewAuthorizationCache(
projectauth.NewReviewer(openshiftClient),
kubeClient.Namespaces(),
policyClient,
)
}
func stopServeHostnameService(c *client.Client, ns, name string) error {
if err := DeleteRC(c, ns, name); err != nil {
return err
}
if err := c.Services(ns).Delete(name); err != nil {
return err
}
return nil
}
// generateSecretsConfig generates any Secret and Volume objects, such
// as SSH private keys, that are necessary for the router container.
func generateSecretsConfig(cfg *RouterConfig, kClient *kclient.Client,
namespace string) ([]*kapi.Secret, []kapi.Volume, []kapi.VolumeMount,
error) {
secrets := []*kapi.Secret{}
volumes := []kapi.Volume{}
mounts := []kapi.VolumeMount{}
if len(cfg.ExternalHostPrivateKey) != 0 {
privkeyData, err := loadKey(cfg.ExternalHostPrivateKey)
if err != nil {
return secrets, volumes, mounts, fmt.Errorf("error reading private key"+
" for external host: %v", err)
}
serviceAccount, err := kClient.ServiceAccounts(namespace).Get(cfg.ServiceAccount)
if err != nil {
return secrets, volumes, mounts, fmt.Errorf("error looking up"+
" service account %s: %v", cfg.ServiceAccount, err)
}
privkeySecret := &kapi.Secret{
ObjectMeta: kapi.ObjectMeta{
Name: privkeySecretName,
Annotations: map[string]string{
kapi.ServiceAccountNameKey: serviceAccount.Name,
kapi.ServiceAccountUIDKey: string(serviceAccount.UID),
},
},
Data: map[string][]byte{privkeyName: privkeyData},
}
secrets = append(secrets, privkeySecret)
}
// We need a secrets volume and mount iff we have secrets.
if len(secrets) != 0 {
secretsVolume := kapi.Volume{
Name: secretsVolumeName,
VolumeSource: kapi.VolumeSource{
Secret: &kapi.SecretVolumeSource{
SecretName: privkeySecretName,
},
},
}
secretsMount := kapi.VolumeMount{
Name: secretsVolumeName,
ReadOnly: true,
MountPath: secretsPath,
}
volumes = []kapi.Volume{secretsVolume}
mounts = []kapi.VolumeMount{secretsMount}
}
return secrets, volumes, mounts, nil
}
func resizeRC(c *client.Client, ns, name string, replicas int) error {
rc, err := c.ReplicationControllers(ns).Get(name)
if err != nil {
return err
}
rc.Spec.Replicas = replicas
_, err = c.ReplicationControllers(rc.Namespace).Update(rc)
return err
}
func newPodOnNode(c *client.Client, namespace, podName, nodeName string) error {
pod, err := c.Pods(namespace).Create(podOnNode(podName, nodeName, serveHostnameImage))
if err == nil {
Logf("Created pod %s on node %s", pod.ObjectMeta.Name, nodeName)
} else {
Logf("Failed to create pod %s on node %s: %v", podName, nodeName, err)
}
return err
}
func countRemaining(c *client.Client, withName string) (int, error) {
var cnt = 0
nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything())
for _, item := range nsList.Items {
if strings.Contains(item.Name, "nslifetest") {
cnt++
}
}
return cnt, err
}
func LoadExistingNextReplicationController(c *client.Client, namespace, newName string) (*api.ReplicationController, error) {
if len(newName) == 0 {
return nil, nil
}
newRc, err := c.ReplicationControllers(namespace).Get(newName)
if err != nil && errors.IsNotFound(err) {
return nil, nil
}
return newRc, err
}
func runSchedulerNoPhantomPodsTest(client *client.Client) {
pod := &api.Pod{
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "c1",
Image: "kubernetes/pause",
Ports: []api.ContainerPort{
{ContainerPort: 1234, HostPort: 9999},
},
ImagePullPolicy: api.PullIfNotPresent,
},
},
},
}
// Assuming we only have two kublets, the third pod here won't schedule
// if the scheduler doesn't correctly handle the delete for the second
// pod.
pod.ObjectMeta.Name = "phantom.foo"
foo, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, time.Second*30, podRunning(client, foo.Namespace, foo.Name)); err != nil {
glog.Fatalf("FAILED: pod never started running %v", err)
}
pod.ObjectMeta.Name = "phantom.bar"
bar, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, time.Second*30, podRunning(client, bar.Namespace, bar.Name)); err != nil {
glog.Fatalf("FAILED: pod never started running %v", err)
}
// Delete a pod to free up room.
glog.Infof("Deleting pod %v", bar.Name)
err = client.Pods(api.NamespaceDefault).Delete(bar.Name, nil)
if err != nil {
glog.Fatalf("FAILED: couldn't delete pod %q: %v", bar.Name, err)
}
pod.ObjectMeta.Name = "phantom.baz"
baz, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, time.Second*60, podRunning(client, baz.Namespace, baz.Name)); err != nil {
glog.Fatalf("FAILED: (Scheduler probably didn't process deletion of 'phantom.bar') Pod never started running: %v", err)
}
glog.Info("Scheduler doesn't make phantom pods: test passed.")
}
func getAllNodesInCluster(c *client.Client) ([]string, error) {
nodeList, err := c.Nodes().List(labels.Everything(), fields.Everything())
if err != nil {
return nil, err
}
result := []string{}
for _, node := range nodeList.Items {
result = append(result, node.Name)
}
return result, nil
}
func validateTemplateService(c *k8sclient.Client, f *cmdutil.Factory) (Result, error) {
ns, _, err := f.DefaultNamespace()
if err != nil {
return Failure, err
}
svc, err := c.Services(ns).Get("templates")
if svc != nil {
return Success, err
}
return Failure, err
}
func validateConsoleDeployment(c *k8sclient.Client, f *cmdutil.Factory) (Result, error) {
ns, _, err := f.DefaultNamespace()
if err != nil {
return Failure, err
}
rc, err := c.ReplicationControllers(ns).Get("fabric8")
if rc != nil {
return Success, err
}
return Failure, err
}
func validateServiceAccount(c *k8sclient.Client, f *cmdutil.Factory) (Result, error) {
ns, _, err := f.DefaultNamespace()
if err != nil {
return Failure, err
}
sa, err := c.ServiceAccounts(ns).Get("fabric8")
if sa != nil {
return Success, err
}
return Failure, err
}
func createSecret(c *k8sclient.Client, f *cmdutil.Factory, flags *flag.FlagSet, secretDataIdentifiers string, secretType string, keysNames []string) (Result, error) {
var secret = secret(secretDataIdentifiers, secretType, keysNames, flags)
ns, _, err := f.DefaultNamespace()
if err != nil {
return Failure, err
}
rs, err := c.Secrets(ns).Create(&secret)
if rs != nil {
return Success, err
}
return Failure, err
}
func getMinionPublicIps(c *client.Client) ([]string, error) {
nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
if err != nil {
return nil, err
}
ips := collectAddresses(nodes, api.NodeExternalIP)
if len(ips) == 0 {
ips = collectAddresses(nodes, api.NodeLegacyHostIP)
}
return ips, nil
}
func getReferencedServiceAccountToken(c *client.Client, ns string, name string, shouldWait bool) (string, string, error) {
tokenName := ""
token := ""
findToken := func() (bool, error) {
user, err := c.ServiceAccounts(ns).Get(name)
if errors.IsNotFound(err) {
return false, nil
}
if err != nil {
return false, err
}
for _, ref := range user.Secrets {
secret, err := c.Secrets(ns).Get(ref.Name)
if errors.IsNotFound(err) {
continue
}
if err != nil {
return false, err
}
if secret.Type != api.SecretTypeServiceAccountToken {
continue
}
name := secret.Annotations[api.ServiceAccountNameKey]
uid := secret.Annotations[api.ServiceAccountUIDKey]
tokenData := secret.Data[api.ServiceAccountTokenKey]
if name == user.Name && uid == string(user.UID) && len(tokenData) > 0 {
tokenName = secret.Name
token = string(tokenData)
return true, nil
}
}
return false, nil
}
if shouldWait {
err := wait.Poll(time.Second, 10*time.Second, findToken)
if err != nil {
return "", "", err
}
} else {
ok, err := findToken()
if err != nil {
return "", "", err
}
if !ok {
return "", "", fmt.Errorf("No token found for %s/%s", ns, name)
}
}
return tokenName, token, nil
}
// StartRC creates given rc if it doesn't already exist, then updates it via kubectl's scaler.
func StartRC(controller *api.ReplicationController, restClient *client.Client) (*api.ReplicationController, error) {
created, err := restClient.ReplicationControllers(controller.Namespace).Get(controller.Name)
if err != nil {
glog.Infof("Rc %v doesn't exist, creating", controller.Name)
created, err = restClient.ReplicationControllers(controller.Namespace).Create(controller)
if err != nil {
return nil, err
}
}
// If we just created an rc, wait till it creates its replicas.
return ScaleRC(created.Name, created.Namespace, controller.Spec.Replicas, restClient)
}
func getServiceAccountPullSecret(client *kclient.Client, ns, name string) (string, error) {
secrets, err := client.Secrets(ns).List(labels.Everything(), fields.Everything())
if err != nil {
return "", err
}
for _, secret := range secrets.Items {
if secret.Type == api.SecretTypeDockercfg && secret.Annotations[api.ServiceAccountNameKey] == name {
return string(secret.Data[api.DockerConfigKey]), nil
}
}
return "", nil
}
// Clean both server and client pods.
func volumeTestCleanup(client *client.Client, config VolumeTestConfig) {
By(fmt.Sprint("cleaning the environment after ", config.prefix))
defer GinkgoRecover()
podClient := client.Pods(config.namespace)
serviceClient := client.Services(config.namespace)
// ignore all errors, the pods may not be even created
podClient.Delete(config.prefix+"-client", nil)
serviceClient.Delete(config.prefix + "-server")
podClient.Delete(config.prefix+"-server", nil)
}