func CheckCadvisorHealthOnAllNodes(c *client.Client, timeout time.Duration) {
By("getting list of nodes")
nodeList, err := c.Nodes().List(labels.Everything(), fields.Everything())
expectNoError(err)
var errors []error
retries := maxRetries
for {
errors = []error{}
for _, node := range nodeList.Items {
// cadvisor is not accessible directly unless its port (4194 by default) is exposed.
// Here, we access '/stats/' REST endpoint on the qinglet which polls cadvisor internally.
statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
By(fmt.Sprintf("Querying stats from node %s using url %s", node.Name, statsResource))
_, err = c.Get().AbsPath(statsResource).Timeout(timeout).Do().Raw()
if err != nil {
errors = append(errors, err)
}
}
if len(errors) == 0 {
return
}
if retries--; retries <= 0 {
break
}
Logf("failed to retrieve qinglet stats -\n %v", errors)
time.Sleep(sleepDuration)
}
Failf("Failed after retrying %d times for cadvisor to be healthy on all nodes. Errors:\n%v", maxRetries, errors)
}
// Retrieve metrics from the qinglet server of the given node.
func getQingletMetrics(c *client.Client, node string) (string, error) {
metric, err := c.Get().
Prefix("proxy").
Resource("nodes").
Name(fmt.Sprintf("%v:%v", node, ports.QingletPort)).
Suffix("metrics").
Do().
Raw()
if err != nil {
return "", err
}
return string(metric), nil
}
func makeRequestToGuestbook(c *client.Client, cmd, value string, ns string) (string, error) {
result, err := c.Get().
Prefix("proxy").
Namespace(ns).
Resource("services").
Name("frontend").
Suffix("/index.php").
Param("cmd", cmd).
Param("key", "messages").
Param("value", value).
Do().
Raw()
return string(result), err
}
func assertFilesExist(fileNames []string, fileDir string, pod *api.Pod, client *client.Client) {
var failed []string
expectNoError(wait.Poll(time.Second*2, time.Second*60, func() (bool, error) {
failed = []string{}
for _, fileName := range fileNames {
if _, err := client.Get().
Prefix("proxy").
Resource("pods").
Namespace(pod.Namespace).
Name(pod.Name).
Suffix(fileDir, fileName).
Do().Raw(); err != nil {
Logf("Unable to read %s from pod %s: %v", fileName, pod.Name, err)
failed = append(failed, fileName)
}
}
if len(failed) == 0 {
return true, nil
}
Logf("Lookups using %s failed for: %v\n", pod.Name, failed)
return false, nil
}))
Expect(len(failed)).To(Equal(0))
}
// NewListWatchFromClient creates a new ListWatch from the specified client, resource, namespace and field selector.
func NewListWatchFromClient(c *client.Client, resource string, namespace string, fieldSelector fields.Selector) *ListWatch {
listFunc := func() (runtime.Object, error) {
return c.Get().
Namespace(namespace).
Resource(resource).
FieldsSelectorParam(fieldSelector).
Do().
Get()
}
watchFunc := func(resourceVersion string) (watch.Interface, error) {
return c.Get().
Prefix("watch").
Namespace(namespace).
Resource(resource).
FieldsSelectorParam(fieldSelector).
Param("resourceVersion", resourceVersion).Watch()
}
return &ListWatch{ListFunc: listFunc, WatchFunc: watchFunc}
}
// Start a client pod using given VolumeSource (exported by startVolumeServer())
// and check that the pod sees the data from the server pod.
func testVolumeClient(client *client.Client, config VolumeTestConfig, volume api.VolumeSource, expectedContent string) {
By(fmt.Sprint("starting ", config.prefix, " client"))
podClient := client.Pods(config.namespace)
clientPod := &api.Pod{
TypeMeta: api.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: api.ObjectMeta{
Name: config.prefix + "-client",
Labels: map[string]string{
"role": config.prefix + "-client",
},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: config.prefix + "-client",
Image: "qingyuan/nginx:1.7.9",
Ports: []api.ContainerPort{
{
Name: "web",
ContainerPort: 80,
Protocol: api.ProtocolTCP,
},
},
VolumeMounts: []api.VolumeMount{
{
Name: config.prefix + "-volume",
MountPath: "/usr/share/nginx/html",
},
},
},
},
Volumes: []api.Volume{
{
Name: config.prefix + "-volume",
VolumeSource: volume,
},
},
},
}
if _, err := podClient.Create(clientPod); err != nil {
Failf("Failed to create %s pod: %v", clientPod.Name, err)
}
expectNoError(waitForPodRunningInNamespace(client, clientPod.Name, config.namespace))
By("reading a web page from the client")
body, err := client.Get().
Namespace(config.namespace).
Prefix("proxy").
Resource("pods").
Name(clientPod.Name).
DoRaw()
expectNoError(err, "Cannot read web page: %v", err)
Logf("body: %v", string(body))
By("checking the page content")
Expect(body).To(ContainSubstring(expectedContent))
}
func testPreStop(c *client.Client, ns string) {
// This is the server that will receive the preStop notification
podDescr := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: "server",
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "server",
Image: "qingyuan/nettest:1.3",
Ports: []api.ContainerPort{{ContainerPort: 8080}},
},
},
},
}
By(fmt.Sprintf("Creating server pod %s in namespace %s", podDescr.Name, ns))
_, err := c.Pods(ns).Create(podDescr)
expectNoError(err, fmt.Sprintf("creating pod %s", podDescr.Name))
// At the end of the test, clean up by removing the pod.
defer func() {
By("Deleting the server pod")
c.Pods(ns).Delete(podDescr.Name, nil)
}()
By("Waiting for pods to come up.")
err = waitForPodRunningInNamespace(c, podDescr.Name, ns)
expectNoError(err, "waiting for server pod to start")
val := "{\"Source\": \"prestop\"}"
podOut, err := c.Pods(ns).Get(podDescr.Name)
expectNoError(err, "getting pod info")
preStopDescr := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: "tester",
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "tester",
Image: "qingyuan/busybox",
Command: []string{"sleep", "600"},
Lifecycle: &api.Lifecycle{
PreStop: &api.Handler{
Exec: &api.ExecAction{
Command: []string{
"wget", "-O-", "--post-data=" + val, fmt.Sprintf("http://%s:8080/write", podOut.Status.PodIP),
},
},
},
},
},
},
},
}
By(fmt.Sprintf("Creating tester pod %s in namespace %s", podDescr.Name, ns))
_, err = c.Pods(ns).Create(preStopDescr)
expectNoError(err, fmt.Sprintf("creating pod %s", preStopDescr.Name))
deletePreStop := true
// At the end of the test, clean up by removing the pod.
defer func() {
if deletePreStop {
By("Deleting the tester pod")
c.Pods(ns).Delete(preStopDescr.Name, nil)
}
}()
err = waitForPodRunningInNamespace(c, preStopDescr.Name, ns)
expectNoError(err, "waiting for tester pod to start")
// Delete the pod with the preStop handler.
By("Deleting pre-stop pod")
if err := c.Pods(ns).Delete(preStopDescr.Name, nil); err == nil {
deletePreStop = false
}
expectNoError(err, fmt.Sprintf("deleting pod: %s", preStopDescr.Name))
// Validate that the server received the web poke.
err = wait.Poll(time.Second*5, time.Second*60, func() (bool, error) {
if body, err := c.Get().
Namespace(ns).Prefix("proxy").
Resource("pods").
Name(podDescr.Name).
Suffix("read").
DoRaw(); err != nil {
By(fmt.Sprintf("Error validating prestop: %v", err))
} else {
Logf("Saw: %s", string(body))
state := State{}
err := json.Unmarshal(body, &state)
if err != nil {
Logf("Error parsing: %v", err)
return false, nil
}
if state.Received["prestop"] != 0 {
return true, nil
}
}
return false, nil
})
expectNoError(err, "validating pre-stop.")
}
func runSelfLinkTestOnNamespace(c *client.Client, namespace string) {
svcBody := api.Service{
ObjectMeta: api.ObjectMeta{
Name: "selflinktest",
Namespace: namespace,
Labels: map[string]string{
"name": "selflinktest",
},
},
Spec: api.ServiceSpec{
// This is here because validation requires it.
Selector: map[string]string{
"foo": "bar",
},
Ports: []api.ServicePort{{
Port: 12345,
Protocol: "TCP",
}},
SessionAffinity: "None",
},
}
services := c.Services(namespace)
svc, err := services.Create(&svcBody)
if err != nil {
glog.Fatalf("Failed creating selflinktest service: %v", err)
}
err = c.Get().RequestURI(svc.SelfLink).Do().Into(svc)
if err != nil {
glog.Fatalf("Failed listing service with supplied self link '%v': %v", svc.SelfLink, err)
}
svcList, err := services.List(labels.Everything())
if err != nil {
glog.Fatalf("Failed listing services: %v", err)
}
err = c.Get().RequestURI(svcList.SelfLink).Do().Into(svcList)
if err != nil {
glog.Fatalf("Failed listing services with supplied self link '%v': %v", svcList.SelfLink, err)
}
found := false
for i := range svcList.Items {
item := &svcList.Items[i]
if item.Name != "selflinktest" {
continue
}
found = true
err = c.Get().RequestURI(item.SelfLink).Do().Into(svc)
if err != nil {
glog.Fatalf("Failed listing service with supplied self link '%v': %v", item.SelfLink, err)
}
break
}
if !found {
glog.Fatalf("never found selflinktest service in namespace %s", namespace)
}
glog.Infof("Self link test passed in namespace %s", namespace)
// TODO: Should test PUT at some point, too.
}