func ensureNamespaceExists(c *k8sclient.Client, oc *oclient.Client, ns string) error {
typeOfMaster := util.TypeOfMaster(c)
if typeOfMaster == util.Kubernetes {
nss := c.Namespaces()
_, err := nss.Get(ns)
if err != nil {
// lets assume it doesn't exist!
util.Infof("Creating new Namespace: %s\n", ns)
entity := kapi.Namespace{
ObjectMeta: kapi.ObjectMeta{Name: ns},
}
_, err := nss.Create(&entity)
return err
}
} else {
_, err := oc.Projects().Get(ns)
if err != nil {
// lets assume it doesn't exist!
request := projectapi.ProjectRequest{
ObjectMeta: kapi.ObjectMeta{Name: ns},
}
util.Infof("Creating new Project: %s\n", ns)
_, err := oc.ProjectRequests().Create(&request)
return err
}
}
return nil
}
func newProjectAuthorizationCache(authorizer authorizer.Authorizer, kubeClient *kclient.Client, policyClient policyclient.ReadOnlyPolicyClient) *projectauth.AuthorizationCache {
return projectauth.NewAuthorizationCache(
projectauth.NewAuthorizerReviewer(authorizer),
kubeClient.Namespaces(),
policyClient,
)
}
func newProjectAuthorizationCache(openshiftClient *osclient.Client, kubeClient *kclient.Client,
policyClient policyclient.ReadOnlyPolicyClient) *projectauth.AuthorizationCache {
return projectauth.NewAuthorizationCache(
projectauth.NewReviewer(openshiftClient),
kubeClient.Namespaces(),
policyClient,
)
}
func countRemaining(c *client.Client, withName string) (int, error) {
var cnt = 0
nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything())
for _, item := range nsList.Items {
if strings.Contains(item.Name, "nslifetest") {
cnt++
}
}
return cnt, err
}
func newProjectAuthorizationCache(authorizer authorizer.Authorizer, kubeClient *kclient.Client, informerFactory shared.InformerFactory) *projectauth.AuthorizationCache {
return projectauth.NewAuthorizationCache(
projectauth.NewAuthorizerReviewer(authorizer),
kubeClient.Namespaces(),
informerFactory.ClusterPolicies().Lister(),
informerFactory.ClusterPolicyBindings().Lister(),
informerFactory.Policies().Lister(),
informerFactory.PolicyBindings().Lister(),
)
}
func createKubeSystemIfNotPresent(client *kubeclient.Client) error {
var ns = &kube.Namespace{
ObjectMeta: kube.ObjectMeta{
Name: DNSServiceNamespace,
},
}
_, err := client.Namespaces().Create(ns)
if err != nil {
return err
}
return nil
}
func deleteEnvironments(c *k8sclient.Client, selector labels.Selector) error {
ns, err := c.Namespaces().List(api.ListOptions{LabelSelector: selector})
if err != nil {
return err
}
for _, n := range ns.Items {
err := c.Namespaces().Delete(n.Name)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete Namespace %s", n.Name))
}
}
return nil
}
func extinguish(c *client.Client, totalNS int, maxAllowedAfterDel int, maxSeconds int) {
var err error
By("Creating testing namespaces")
wg := &sync.WaitGroup{}
for n := 0; n < totalNS; n += 1 {
wg.Add(1)
go func(n int) {
defer wg.Done()
defer GinkgoRecover()
_, err = createTestingNS(fmt.Sprintf("nslifetest-%v", n), c)
Expect(err).NotTo(HaveOccurred())
}(n)
}
wg.Wait()
By("Waiting 10 seconds")
//Wait 10 seconds, then SEND delete requests for all the namespaces.
time.Sleep(time.Duration(10 * time.Second))
By("Deleting namespaces")
nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything())
Expect(err).NotTo(HaveOccurred())
var nsCount = 0
for _, item := range nsList.Items {
if strings.Contains(item.Name, "nslifetest") {
wg.Add(1)
nsCount++
go func(nsName string) {
defer wg.Done()
defer GinkgoRecover()
Expect(c.Namespaces().Delete(nsName)).To(Succeed())
Logf("namespace : %v api call to delete is complete ", nsName)
}(item.Name)
}
}
Expect(nsCount).To(Equal(totalNS))
wg.Wait()
By("Waiting for namespaces to vanish")
//Now POLL until all namespaces have been eradicated.
expectNoError(wait.Poll(2*time.Second, time.Duration(maxSeconds)*time.Second,
func() (bool, error) {
if rem, err := countRemaining(c, "nslifetest"); err != nil || rem > maxAllowedAfterDel {
Logf("Remaining namespaces : %v", rem)
return false, err
} else {
return true, nil
}
}))
}
func GetClientForServiceAccount(adminClient *kclient.Client, clientConfig kclient.Config, namespace, name string) (*client.Client, *kclient.Client, *kclient.Config, error) {
_, err := adminClient.Namespaces().Create(&kapi.Namespace{ObjectMeta: kapi.ObjectMeta{Name: namespace}})
if err != nil && !kerrs.IsAlreadyExists(err) {
return nil, nil, nil, err
}
sa, err := adminClient.ServiceAccounts(namespace).Create(&kapi.ServiceAccount{ObjectMeta: kapi.ObjectMeta{Name: name}})
if kerrs.IsAlreadyExists(err) {
sa, err = adminClient.ServiceAccounts(namespace).Get(name)
}
if err != nil {
return nil, nil, nil, err
}
token := ""
err = wait.Poll(time.Second, 30*time.Second, func() (bool, error) {
selector := fields.OneTermEqualSelector(kclient.SecretType, string(kapi.SecretTypeServiceAccountToken))
secrets, err := adminClient.Secrets(namespace).List(kapi.ListOptions{FieldSelector: selector})
if err != nil {
return false, err
}
for _, secret := range secrets.Items {
if serviceaccounts.IsValidServiceAccountToken(sa, &secret) {
token = string(secret.Data[kapi.ServiceAccountTokenKey])
return true, nil
}
}
return false, nil
})
if err != nil {
return nil, nil, nil, err
}
saClientConfig := clientcmd.AnonymousClientConfig(clientConfig)
saClientConfig.BearerToken = token
kubeClient, err := kclient.New(&saClientConfig)
if err != nil {
return nil, nil, nil, err
}
osClient, err := client.New(&saClientConfig)
if err != nil {
return nil, nil, nil, err
}
return osClient, kubeClient, &saClientConfig, nil
}
func extinguish(c *client.Client, totalNS int, maxAllowedAfterDel int, maxSeconds int) {
var err error
By("Creating testing namespaces")
wg := &sync.WaitGroup{}
for n := 0; n < totalNS; n += 1 {
wg.Add(1)
go func(n int) {
defer wg.Done()
defer GinkgoRecover()
_, err = createTestingNS(fmt.Sprintf("nslifetest-%v", n), c, nil)
Expect(err).NotTo(HaveOccurred())
}(n)
}
wg.Wait()
//Wait 10 seconds, then SEND delete requests for all the namespaces.
By("Waiting 10 seconds")
time.Sleep(time.Duration(10 * time.Second))
deleted, err := deleteNamespaces(c, []string{"nslifetest"}, nil /* skipFilter */)
Expect(err).NotTo(HaveOccurred())
Expect(len(deleted)).To(Equal(totalNS))
By("Waiting for namespaces to vanish")
//Now POLL until all namespaces have been eradicated.
expectNoError(wait.Poll(2*time.Second, time.Duration(maxSeconds)*time.Second,
func() (bool, error) {
var cnt = 0
nsList, err := c.Namespaces().List(api.ListOptions{})
if err != nil {
return false, err
}
for _, item := range nsList.Items {
if strings.Contains(item.Name, "nslifetest") {
cnt++
}
}
if cnt > maxAllowedAfterDel {
Logf("Remaining namespaces : %v", cnt)
return false, nil
}
return true, nil
}))
}
// Creates namespace based on given specification.
func CreateNamespace(spec *NamespaceSpec, client *client.Client) error {
namespace := &api.Namespace{
ObjectMeta: api.ObjectMeta{
Name: spec.Name,
},
}
_, err := client.Namespaces().Create(namespace)
return err
}
func DeleteAndWaitForNamespaceTermination(c *kclient.Client, name string) error {
w, err := c.Namespaces().Watch(kapi.ListOptions{})
if err != nil {
return err
}
if err := c.Namespaces().Delete(name); err != nil {
return err
}
_, err = cmd.Until(30*time.Second, w, func(event watch.Event) (bool, error) {
if event.Type != watch.Deleted {
return false, nil
}
namespace, ok := event.Object.(*kapi.Namespace)
if !ok {
return false, nil
}
return namespace.Name == name, nil
})
return err
}
// CreateNamespace creates namespace based on given specification.
func CreateNamespace(spec *NamespaceSpec, client *client.Client) error {
log.Printf("Creating namespace %s", spec.Name)
namespace := &api.Namespace{
ObjectMeta: api.ObjectMeta{
Name: spec.Name,
},
}
_, err := client.Namespaces().Create(namespace)
return err
}
func runReplicationControllerTest(c *client.Client) {
clientAPIVersion := c.APIVersion()
data, err := ioutil.ReadFile("cmd/integration/" + clientAPIVersion + "-controller.json")
if err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
var controller api.ReplicationController
if err := api.Scheme.DecodeInto(data, &controller); err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
glog.Infof("Creating replication controllers")
c.Tenants().Create(&api.Tenant{ObjectMeta: api.ObjectMeta{Name: "test"}})
c.Namespaces().Create(&api.Namespace{ObjectMeta: api.ObjectMeta{Name: "test", Tenant: "test"}})
updated, err := c.ReplicationControllers("test").Create(&controller)
if err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
glog.Infof("Done creating replication controllers")
// In practice the controller doesn't need 60s to create a handful of pods, but network latencies on CI
// systems have been observed to vary unpredictably, so give the controller enough time to create pods.
// Our e2e scalability tests will catch controllers that are *actually* slow.
if err := wait.Poll(time.Second, longTestTimeout, client.ControllerHasDesiredReplicas(c, updated)); err != nil {
glog.Fatalf("FAILED: pods never created %v", err)
}
// Poll till we can retrieve the status of all pods matching the given label selector from their nodes.
// This involves 3 operations:
// - The scheduler must assign all pods to a node
// - The assignment must reflect in a `List` operation against the apiserver, for labels matching the selector
// - We need to be able to query the kubelet on that node for information about the pod
if err := wait.Poll(
time.Second, longTestTimeout, podsOnNodes(c, "test", labels.Set(updated.Spec.Selector).AsSelector())); err != nil {
glog.Fatalf("FAILED: pods never started running %v", err)
}
glog.Infof("Pods created")
}
// Returns a list of all namespaces in the cluster.
func GetNamespaceList(client *client.Client) (*NamespacesList, error) {
list, err := client.Namespaces().List(labels.Everything(), fields.Everything())
if err != nil {
return nil, err
}
namespaceList := &NamespacesList{}
for _, element := range list.Items {
namespaceList.Namespaces = append(namespaceList.Namespaces, element.ObjectMeta.Name)
}
return namespaceList, nil
}
// GetNamespaceList returns a list of all namespaces in the cluster.
func GetNamespaceList(client *client.Client, dsQuery *dataselect.DataSelectQuery) (*NamespaceList,
error) {
log.Printf("Getting namespace list")
namespaces, err := client.Namespaces().List(api.ListOptions{
LabelSelector: labels.Everything(),
FieldSelector: fields.Everything(),
})
if err != nil {
return nil, err
}
return toNamespaceList(namespaces.Items, dsQuery), nil
}
// Returns a list of all namespaces in the cluster.
func GetNamespaceList(client *client.Client) (*NamespaceList, error) {
list, err := client.Namespaces().List(unversioned.ListOptions{
LabelSelector: unversioned.LabelSelector{labels.Everything()},
FieldSelector: unversioned.FieldSelector{fields.Everything()},
})
if err != nil {
return nil, err
}
namespaceList := &NamespaceList{}
for _, element := range list.Items {
namespaceList.Namespaces = append(namespaceList.Namespaces, element.ObjectMeta.Name)
}
return namespaceList, nil
}
// Returns a list of all namespaces in the cluster.
func GetNamespaceList(client *client.Client) (*NamespaceList, error) {
log.Printf("Getting namespace list")
list, err := client.Namespaces().List(api.ListOptions{
LabelSelector: labels.Everything(),
FieldSelector: fields.Everything(),
})
if err != nil {
return nil, err
}
namespaceList := &NamespaceList{}
for _, element := range list.Items {
namespaceList.Namespaces = append(namespaceList.Namespaces, element.ObjectMeta.Name)
}
return namespaceList, nil
}
func watchProxyTest(cluster1AdminKubeClient, cluster2AdminKubeClient *kclient.Client, t *testing.T) {
// list namespaces in order to determine correct resourceVersion
namespaces, err := cluster1AdminKubeClient.Namespaces().List(kapi.ListOptions{})
// open a watch on Cluster 2 for namespaces starting with latest resourceVersion
namespaceWatch, err := cluster2AdminKubeClient.Namespaces().Watch(kapi.ListOptions{ResourceVersion: namespaces.ResourceVersion})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer namespaceWatch.Stop()
// add namespace in Cluster 2
namespace := &kapi.Namespace{
ObjectMeta: kapi.ObjectMeta{Name: "test-namespace"},
}
createdNamespace, err := cluster2AdminKubeClient.Namespaces().Create(namespace)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// consume watch output and record it if it's the event we want to see
select {
case e := <-namespaceWatch.ResultChan():
// check that the watch shows the new namespace
if e.Type != watch.Added {
t.Fatalf("expected an Added event but got: %v", e)
}
addedNamespace, ok := e.Object.(*kapi.Namespace)
if !ok {
t.Fatalf("unexpected cast error from event Object to Namespace")
}
if addedNamespace.ObjectMeta.Name != createdNamespace.Name {
t.Fatalf("namespace returned from Watch is not the same ast that created: got %v, wanted %v", createdNamespace, addedNamespace)
}
case <-time.After(10 * time.Second):
t.Fatal("Timed out waiting for watch")
}
}
var _ = Describe("Examples e2e", func() {
var c *client.Client
var ns string
var testingNs *api.Namespace
BeforeEach(func() {
var err error
c, err = loadClient()
expectNoError(err)
testingNs, err = createTestingNS("examples", c)
ns = testingNs.Name
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
if err := c.Namespaces().Delete(ns); err != nil {
Failf("Couldn't delete ns %s", err)
}
})
Describe("[Skipped][Example]Redis", func() {
It("should create and stop redis servers", func() {
mkpath := func(file string) string {
return filepath.Join(testContext.RepoRoot, "examples/redis", file)
}
bootstrapYaml := mkpath("redis-master.yaml")
sentinelServiceYaml := mkpath("redis-sentinel-service.yaml")
sentinelControllerYaml := mkpath("redis-sentinel-controller.yaml")
controllerYaml := mkpath("redis-controller.yaml")
bootstrapPodName := "redis-master"
func dumpClusterInfo(f *cmdutil.Factory, cmd *cobra.Command, args []string, out io.Writer) error {
var c *unversioned.Client
var err error
if c, err = f.Client(); err != nil {
return err
}
printer, _, err := kubectl.GetPrinter("json", "")
if err != nil {
return err
}
nodes, err := c.Nodes().List(api.ListOptions{})
if err != nil {
return err
}
if err := printer.PrintObj(nodes, setupOutputWriter(cmd, out, "nodes.json")); err != nil {
return err
}
var namespaces []string
if cmdutil.GetFlagBool(cmd, "all-namespaces") {
namespaceList, err := c.Namespaces().List(api.ListOptions{})
if err != nil {
return err
}
for ix := range namespaceList.Items {
namespaces = append(namespaces, namespaceList.Items[ix].Name)
}
} else {
namespaces = cmdutil.GetFlagStringSlice(cmd, "namespaces")
if len(namespaces) == 0 {
cmdNamespace, _, err := f.DefaultNamespace()
if err != nil {
return err
}
namespaces = []string{
api.NamespaceSystem,
cmdNamespace,
}
}
}
for _, namespace := range namespaces {
// TODO: this is repetitive in the extreme. Use reflection or
// something to make this a for loop.
events, err := c.Events(namespace).List(api.ListOptions{})
if err != nil {
return err
}
if err := printer.PrintObj(events, setupOutputWriter(cmd, out, path.Join(namespace, "events.json"))); err != nil {
return err
}
rcs, err := c.ReplicationControllers(namespace).List(api.ListOptions{})
if err != nil {
return err
}
if err := printer.PrintObj(rcs, setupOutputWriter(cmd, out, path.Join(namespace, "replication-controllers.json"))); err != nil {
return err
}
svcs, err := c.Services(namespace).List(api.ListOptions{})
if err != nil {
return err
}
if err := printer.PrintObj(svcs, setupOutputWriter(cmd, out, path.Join(namespace, "services.json"))); err != nil {
return err
}
sets, err := c.DaemonSets(namespace).List(api.ListOptions{})
if err != nil {
return err
}
if err := printer.PrintObj(sets, setupOutputWriter(cmd, out, path.Join(namespace, "daemonsets.json"))); err != nil {
return err
}
deps, err := c.Deployments(namespace).List(api.ListOptions{})
if err != nil {
return err
}
if err := printer.PrintObj(deps, setupOutputWriter(cmd, out, path.Join(namespace, "deployments.json"))); err != nil {
return err
}
rps, err := c.ReplicaSets(namespace).List(api.ListOptions{})
if err != nil {
return err
}
if err := printer.PrintObj(rps, setupOutputWriter(cmd, out, path.Join(namespace, "replicasets.json"))); err != nil {
return err
}
pods, err := c.Pods(namespace).List(api.ListOptions{})
if err != nil {
return err
}
if err := printer.PrintObj(pods, setupOutputWriter(cmd, out, path.Join(namespace, "pods.json"))); err != nil {
return err
}
for ix := range pods.Items {
pod := &pods.Items[ix]
writer := setupOutputWriter(cmd, out, path.Join(namespace, pod.Name, "logs.txt"))
writer.Write([]byte(fmt.Sprintf("==== START logs for %s/%s ====\n", pod.Namespace, pod.Name)))
request, err := f.LogsForObject(pod, &api.PodLogOptions{})
if err != nil {
return err
}
data, err := request.DoRaw()
if err != nil {
return err
}
writer.Write(data)
writer.Write([]byte(fmt.Sprintf("==== END logs for %s/%s ====\n", pod.Namespace, pod.Name)))
}
}
dir := cmdutil.GetFlagString(cmd, "output-directory")
if len(dir) == 0 {
dir = "."
}
if dir != "-" {
fmt.Fprintf(out, "Cluster info dumped to %s", dir)
}
return nil
}
// during the test so clean it up here
rc, err := c.ReplicationControllers(ns).Get(RCName)
if err == nil && rc.Spec.Replicas != 0 {
By("Cleaning up the replication controller")
err := DeleteRC(c, ns, RCName)
expectNoError(err)
}
By("Removing additional pods if any")
for i := 1; i <= minionCount; i++ {
name := additionalPodsPrefix + "-" + strconv.Itoa(i)
c.Pods(ns).Delete(name, nil)
}
By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
if err := c.Namespaces().Delete(ns); err != nil {
Failf("Couldn't delete ns %s", err)
}
expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after"))
// Verify latency metrics
highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events"))
expectNoError(err)
Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests")
})
// Tests with "Skipped" substring in their name will be skipped when running
// e2e test suite without --ginkgo.focus & --ginkgo.skip flags.
type Density struct {
skip bool
func runServiceTest(client *client.Client) {
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Labels: map[string]string{
"name": "thisisalonglabel",
},
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "c1",
Image: "foo",
Ports: []api.ContainerPort{
{ContainerPort: 1234},
},
ImagePullPolicy: api.PullIfNotPresent,
},
},
RestartPolicy: api.RestartPolicyAlways,
DNSPolicy: api.DNSClusterFirst,
},
Status: api.PodStatus{
PodIP: "1.2.3.4",
},
}
pod, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, longTestTimeout, podExists(client, pod.Namespace, pod.Name)); err != nil {
glog.Fatalf("FAILED: pod never started running %v", err)
}
svc1 := &api.Service{
ObjectMeta: api.ObjectMeta{Name: "service1"},
Spec: api.ServiceSpec{
Selector: map[string]string{
"name": "thisisalonglabel",
},
Ports: []api.ServicePort{{
Port: 8080,
Protocol: "TCP",
}},
SessionAffinity: "None",
},
}
svc1, err = client.Services(api.NamespaceDefault).Create(svc1)
if err != nil {
glog.Fatalf("Failed to create service: %v, %v", svc1, err)
}
// create an identical service in the non-default namespace
svc3 := &api.Service{
ObjectMeta: api.ObjectMeta{Name: "service1"},
Spec: api.ServiceSpec{
Selector: map[string]string{
"name": "thisisalonglabel",
},
Ports: []api.ServicePort{{
Port: 8080,
Protocol: "TCP",
}},
SessionAffinity: "None",
},
}
client.Tenants().Create(&api.Tenant{ObjectMeta: api.ObjectMeta{Name: "other"}})
client.Namespaces().Create(&api.Namespace{ObjectMeta: api.ObjectMeta{Name: "other", Tenant: "other"}})
svc3, err = client.Services("other").Create(svc3)
if err != nil {
glog.Fatalf("Failed to create service: %v, %v", svc3, err)
}
// TODO Reduce the timeouts in this test when endpoints controller is sped up. See #6045.
if err := wait.Poll(time.Second, longTestTimeout, endpointsSet(client, svc1.Namespace, svc1.Name, 1)); err != nil {
glog.Fatalf("FAILED: unexpected endpoints: %v", err)
}
// A second service with the same port.
svc2 := &api.Service{
ObjectMeta: api.ObjectMeta{Name: "service2"},
Spec: api.ServiceSpec{
Selector: map[string]string{
"name": "thisisalonglabel",
},
Ports: []api.ServicePort{{
Port: 8080,
Protocol: "TCP",
}},
SessionAffinity: "None",
},
}
svc2, err = client.Services(api.NamespaceDefault).Create(svc2)
if err != nil {
glog.Fatalf("Failed to create service: %v, %v", svc2, err)
}
if err := wait.Poll(time.Second, longTestTimeout, endpointsSet(client, svc2.Namespace, svc2.Name, 1)); err != nil {
glog.Fatalf("FAILED: unexpected endpoints: %v", err)
}
if err := wait.Poll(time.Second, longTestTimeout, endpointsSet(client, svc3.Namespace, svc3.Name, 0)); err != nil {
glog.Fatalf("FAILED: service in other namespace should have no endpoints: %v", err)
}
svcList, err := client.Services(api.NamespaceAll).List(labels.Everything(), fields.Everything())
if err != nil {
glog.Fatalf("Failed to list services across namespaces: %v", err)
}
names := sets.NewString()
for _, svc := range svcList.Items {
names.Insert(fmt.Sprintf("%s/%s", svc.Namespace, svc.Name))
}
if !names.HasAll("default/kubernetes", "default/service1", "default/service2", "other/service1") {
glog.Fatalf("Unexpected service list: %#v", names)
}
glog.Info("Service test passed.")
}