func newClusterClientset(c *v1beta1.Cluster) (*release_1_4.Clientset, error) {
clusterConfig, err := util.BuildClusterConfig(c)
if clusterConfig != nil {
clientset := release_1_4.NewForConfigOrDie(restclient.AddUserAgent(clusterConfig, UserAgentName))
return clientset, nil
}
return nil, err
}
func runStorageTest(t *testing.T, ns string, autoscalingVersion, batchVersion, extensionsVersion unversioned.GroupVersion) {
etcdServer := testutil.RequireEtcd(t)
masterConfig, err := testserver.DefaultMasterOptions()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
keys := etcd.NewKeysAPI(etcdServer.Client)
getGVKFromEtcd := func(prefix, name string) (*unversioned.GroupVersionKind, error) {
key := path.Join(masterConfig.EtcdStorageConfig.KubernetesStoragePrefix, prefix, ns, name)
resp, err := keys.Get(context.TODO(), key, nil)
if err != nil {
return nil, err
}
_, gvk, err := runtime.UnstructuredJSONScheme.Decode([]byte(resp.Node.Value), nil, nil)
return gvk, err
}
// TODO: Set storage versions for API groups
// masterConfig.EtcdStorageConfig.StorageVersions[autoscaling.GroupName] = autoscalingVersion.String()
// masterConfig.EtcdStorageConfig.StorageVersions[batch.GroupName] = batchVersion.String()
// masterConfig.EtcdStorageConfig.StorageVersions[extensions.GroupName] = extensionsVersion.String()
clusterAdminKubeConfig, err := testserver.StartConfiguredMaster(masterConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminClient, err := testutil.GetClusterAdminClient(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminClientConfig, err := testutil.GetClusterAdminClientConfig(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// create the containing project
if _, err := testserver.CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, ns, "admin"); err != nil {
t.Fatalf("unexpected error creating the project: %v", err)
}
projectAdminClient, projectAdminKubeClient, projectAdminKubeConfig, err := testutil.GetClientForUser(*clusterAdminClientConfig, "admin")
if err != nil {
t.Fatalf("unexpected error getting project admin client: %v", err)
}
projectAdminKubeClient14 := kclientset14.NewForConfigOrDie(projectAdminKubeConfig)
if err := testutil.WaitForPolicyUpdate(projectAdminClient, ns, "get", extensions.Resource("horizontalpodautoscalers"), true); err != nil {
t.Fatalf("unexpected error waiting for policy update: %v", err)
}
jobTestcases := map[string]struct {
creator kbatchclient.JobInterface
}{
"batch": {creator: projectAdminKubeClient.Batch().Jobs(ns)},
}
for name, testcase := range jobTestcases {
job := batch.Job{
ObjectMeta: kapi.ObjectMeta{Name: name + "-job"},
Spec: batch.JobSpec{
Template: kapi.PodTemplateSpec{
Spec: kapi.PodSpec{
RestartPolicy: kapi.RestartPolicyNever,
Containers: []kapi.Container{{Name: "containername", Image: "containerimage"}},
},
},
},
}
// Create a Job
if _, err := testcase.creator.Create(&job); err != nil {
t.Fatalf("%s: unexpected error creating Job: %v", name, err)
}
// Ensure it is persisted correctly
if gvk, err := getGVKFromEtcd("jobs", job.Name); err != nil {
t.Fatalf("%s: unexpected error reading Job: %v", name, err)
} else if *gvk != batchVersion.WithKind("Job") {
t.Fatalf("%s: expected api version %s in etcd, got %s reading Job", name, batchVersion, gvk)
}
// Ensure it is accessible from both APIs
if _, err := projectAdminKubeClient.Batch().Jobs(ns).Get(job.Name); err != nil {
t.Errorf("%s: Error reading Job from the batch client: %#v", name, err)
}
if _, err := projectAdminKubeClient14.Extensions().Jobs(ns).Get(job.Name); err != nil {
t.Errorf("%s: Error reading Job from the extensions client: %#v", name, err)
}
}
legacyClient := legacyExtensionsAutoscaling{
projectAdminKubeClient.Autoscaling().HorizontalPodAutoscalers(ns),
projectAdminKubeClient.AutoscalingClient.RESTClient,
ns,
}
hpaTestcases := map[string]struct {
creator kautoscalingclient.HorizontalPodAutoscalerInterface
}{
"autoscaling": {creator: projectAdminKubeClient.Autoscaling().HorizontalPodAutoscalers(ns)},
"extensions": {
creator: legacyClient,
},
}
for name, testcase := range hpaTestcases {
hpa := autoscaling.HorizontalPodAutoscaler{
ObjectMeta: kapi.ObjectMeta{Name: name + "-hpa"},
Spec: autoscaling.HorizontalPodAutoscalerSpec{
MaxReplicas: 1,
ScaleTargetRef: autoscaling.CrossVersionObjectReference{Kind: "ReplicationController", Name: "myrc"},
},
}
// Create an HPA
if _, err := testcase.creator.Create(&hpa); err != nil {
t.Fatalf("%s: unexpected error creating HPA: %v", name, err)
}
// Make sure it is persisted correctly
if gvk, err := getGVKFromEtcd("horizontalpodautoscalers", hpa.Name); err != nil {
t.Fatalf("%s: unexpected error reading HPA: %v", name, err)
} else if *gvk != autoscalingVersion.WithKind("HorizontalPodAutoscaler") {
t.Fatalf("%s: expected api version %s in etcd, got %s reading HPA", name, autoscalingVersion, gvk)
}
// Make sure it is available from the api
if _, err := projectAdminKubeClient.Autoscaling().HorizontalPodAutoscalers(ns).Get(hpa.Name); err != nil {
t.Errorf("%s: Error reading HPA.autoscaling from the autoscaling/v1 API: %#v", name, err)
}
if _, err := legacyClient.Get(hpa.Name); err != nil {
t.Errorf("%s: Error reading HPA.autoscaling from the extensions/v1beta1 API: %#v", name, err)
}
}
}
// Builds a FederatedInformer for the given federation client and factory.
func NewFederatedInformer(
federationClient federation_release_1_4.Interface,
targetInformerFactory TargetInformerFactory,
clusterLifecycle *ClusterLifecycleHandlerFuncs) FederatedInformer {
federatedInformer := &federatedInformerImpl{
targetInformerFactory: targetInformerFactory,
clientFactory: func(cluster *federation_api.Cluster) (kube_release_1_4.Interface, error) {
clusterConfig, err := BuildClusterConfig(cluster)
if err == nil && clusterConfig != nil {
clientset := kube_release_1_4.NewForConfigOrDie(restclient.AddUserAgent(clusterConfig, userAgentName))
return clientset, nil
}
return nil, err
},
targetInformers: make(map[string]informer),
}
getClusterData := func(name string) []interface{} {
data, err := federatedInformer.GetTargetStore().ListFromCluster(name)
if err != nil {
glog.Errorf("Failed to list %s content: %v", name, err)
return make([]interface{}, 0)
}
return data
}
federatedInformer.clusterInformer.store, federatedInformer.clusterInformer.controller = cache.NewInformer(
&cache.ListWatch{
ListFunc: func(options api.ListOptions) (pkg_runtime.Object, error) {
return federationClient.Federation().Clusters().List(options)
},
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
return federationClient.Federation().Clusters().Watch(options)
},
},
&federation_api.Cluster{},
clusterSyncPeriod,
cache.ResourceEventHandlerFuncs{
DeleteFunc: func(old interface{}) {
oldCluster, ok := old.(*federation_api.Cluster)
if ok {
var data []interface{}
if clusterLifecycle.ClusterUnavailable != nil {
data = getClusterData(oldCluster.Name)
}
federatedInformer.deleteCluster(oldCluster)
if clusterLifecycle.ClusterUnavailable != nil {
clusterLifecycle.ClusterUnavailable(oldCluster, data)
}
}
},
AddFunc: func(cur interface{}) {
curCluster, ok := cur.(*federation_api.Cluster)
if ok && isClusterReady(curCluster) {
federatedInformer.addCluster(curCluster)
if clusterLifecycle.ClusterAvailable != nil {
clusterLifecycle.ClusterAvailable(curCluster)
}
} else {
glog.Errorf("Cluster %v not added. Not of correct type, or cluster not ready.", cur)
}
},
UpdateFunc: func(old, cur interface{}) {
oldCluster, ok := old.(*federation_api.Cluster)
if !ok {
glog.Errorf("Internal error: Cluster %v not updated. Old cluster not of correct type.", old)
return
}
curCluster, ok := cur.(*federation_api.Cluster)
if !ok {
glog.Errorf("Internal error: Cluster %v not updated. New cluster not of correct type.", cur)
return
}
if isClusterReady(oldCluster) != isClusterReady(curCluster) || !reflect.DeepEqual(oldCluster.Spec, curCluster.Spec) || !reflect.DeepEqual(oldCluster.ObjectMeta.Annotations, curCluster.ObjectMeta.Annotations) {
var data []interface{}
if clusterLifecycle.ClusterUnavailable != nil {
data = getClusterData(oldCluster.Name)
}
federatedInformer.deleteCluster(oldCluster)
if clusterLifecycle.ClusterUnavailable != nil {
clusterLifecycle.ClusterUnavailable(oldCluster, data)
}
if isClusterReady(curCluster) {
federatedInformer.addCluster(curCluster)
if clusterLifecycle.ClusterAvailable != nil {
clusterLifecycle.ClusterAvailable(curCluster)
}
}
} else {
glog.V(4).Infof("Cluster %v not updated to %v as ready status and specs are identical", oldCluster, curCluster)
}
},
},
)
return federatedInformer
}
func TestExtensionsAPIDisabledAutoscaleBatchEnabled(t *testing.T) {
const projName = "ext-disabled-batch-enabled-proj"
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
masterConfig, err := testserver.DefaultMasterOptions()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Disable all extensions API versions
// Leave autoscaling/batch APIs enabled
masterConfig.KubernetesMasterConfig.DisabledAPIGroupVersions = map[string][]string{"extensions": {"*"}}
clusterAdminKubeConfig, err := testserver.StartConfiguredMaster(masterConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminClient, err := testutil.GetClusterAdminClient(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminKubeClient, err := testutil.GetClusterAdminKubeClient(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminClientConfig, err := testutil.GetClusterAdminClientConfig(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// create the containing project
if _, err := testserver.CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, projName, "admin"); err != nil {
t.Fatalf("unexpected error creating the project: %v", err)
}
projectAdminClient, projectAdminKubeClient, projectAdminKubeConfig, err := testutil.GetClientForUser(*clusterAdminClientConfig, "admin")
if err != nil {
t.Fatalf("unexpected error getting project admin client: %v", err)
}
projectAdminKubeClient14 := kclientset14.NewForConfigOrDie(projectAdminKubeConfig)
if err := testutil.WaitForPolicyUpdate(projectAdminClient, projName, "get", expapi.Resource("horizontalpodautoscalers"), true); err != nil {
t.Fatalf("unexpected error waiting for policy update: %v", err)
}
validHPA := &autoscaling.HorizontalPodAutoscaler{
ObjectMeta: kapi.ObjectMeta{Name: "myjob"},
Spec: autoscaling.HorizontalPodAutoscalerSpec{
ScaleTargetRef: autoscaling.CrossVersionObjectReference{Name: "foo", Kind: "ReplicationController"},
MaxReplicas: 1,
},
}
validJob := &batch.Job{
ObjectMeta: kapi.ObjectMeta{Name: "myjob"},
Spec: batch.JobSpec{
Template: kapi.PodTemplateSpec{
Spec: kapi.PodSpec{
Containers: []kapi.Container{{Name: "mycontainer", Image: "myimage"}},
RestartPolicy: kapi.RestartPolicyNever,
},
},
},
}
legacyAutoscalers := legacyExtensionsAutoscaling{
projectAdminKubeClient.Autoscaling().HorizontalPodAutoscalers(projName),
projectAdminKubeClient.ExtensionsClient.RESTClient,
projName,
}
// make sure extensions API objects cannot be listed or created
if _, err := legacyAutoscalers.List(kapi.ListOptions{}); !errors.IsNotFound(err) {
t.Fatalf("expected NotFound error listing HPA, got %v", err)
}
if _, err := legacyAutoscalers.Create(validHPA); !errors.IsNotFound(err) {
t.Fatalf("expected NotFound error creating HPA, got %v", err)
}
if _, err := projectAdminKubeClient14.Extensions().Jobs(projName).List(kapi.ListOptions{}); !errors.IsNotFound(err) {
t.Fatalf("expected NotFound error listing jobs, got %v", err)
}
if _, err := projectAdminKubeClient14.Extensions().Jobs(projName).Create(&extensions_v1beta1.Job{}); !errors.IsNotFound(err) {
t.Fatalf("expected NotFound error creating job, got %v", err)
}
// make sure autoscaling and batch API objects can be listed and created
if _, err := projectAdminKubeClient.Autoscaling().HorizontalPodAutoscalers(projName).List(kapi.ListOptions{}); err != nil {
t.Fatalf("unexpected error: %#v", err)
}
if _, err := projectAdminKubeClient.Autoscaling().HorizontalPodAutoscalers(projName).Create(validHPA); err != nil {
t.Fatalf("unexpected error: %#v", err)
}
if _, err := projectAdminKubeClient.Batch().Jobs(projName).List(kapi.ListOptions{}); err != nil {
t.Fatalf("unexpected error: %#v", err)
}
if _, err := projectAdminKubeClient.Batch().Jobs(projName).Create(validJob); err != nil {
t.Fatalf("unexpected error: %#v", err)
}
// Delete the containing project
if err := testutil.DeleteAndWaitForNamespaceTermination(clusterAdminKubeClient, projName); err != nil {
t.Fatalf("unexpected error: %#v", err)
}
// recreate the containing project
if _, err := testserver.CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, projName, "admin"); err != nil {
t.Fatalf("unexpected error creating the project: %v", err)
}
projectAdminClient, projectAdminKubeClient, _, err = testutil.GetClientForUser(*clusterAdminClientConfig, "admin")
if err != nil {
t.Fatalf("unexpected error getting project admin client: %v", err)
}
if err := testutil.WaitForPolicyUpdate(projectAdminClient, projName, "get", expapi.Resource("horizontalpodautoscalers"), true); err != nil {
t.Fatalf("unexpected error waiting for policy update: %v", err)
}
// make sure the created objects got cleaned up by namespace deletion
if hpas, err := projectAdminKubeClient.Autoscaling().HorizontalPodAutoscalers(projName).List(kapi.ListOptions{}); err != nil {
t.Fatalf("unexpected error: %#v", err)
} else if len(hpas.Items) > 0 {
t.Fatalf("expected 0 HPA objects, got %#v", hpas.Items)
}
if jobs, err := projectAdminKubeClient.Batch().Jobs(projName).List(kapi.ListOptions{}); err != nil {
t.Fatalf("unexpected error: %#v", err)
} else if len(jobs.Items) > 0 {
t.Fatalf("expected 0 Job objects, got %#v", jobs.Items)
}
}
func TestExtensionsAPIDisabled(t *testing.T) {
const projName = "ext-disabled-proj"
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
masterConfig, err := testserver.DefaultMasterOptions()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Disable all extensions API versions
masterConfig.KubernetesMasterConfig.DisabledAPIGroupVersions = map[string][]string{"extensions": {"*"}, "autoscaling": {"*"}, "batch": {"*"}}
clusterAdminKubeConfig, err := testserver.StartConfiguredMaster(masterConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminClient, err := testutil.GetClusterAdminClient(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminKubeClient, err := testutil.GetClusterAdminKubeClient(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
clusterAdminClientConfig, err := testutil.GetClusterAdminClientConfig(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// create the containing project
if _, err := testserver.CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, projName, "admin"); err != nil {
t.Fatalf("unexpected error creating the project: %v", err)
}
projectAdminClient, projectAdminKubeClient, projectAdminKubeConfig, err := testutil.GetClientForUser(*clusterAdminClientConfig, "admin")
if err != nil {
t.Fatalf("unexpected error getting project admin client: %v", err)
}
projectAdminKubeClient14 := kclientset14.NewForConfigOrDie(projectAdminKubeConfig)
if err := testutil.WaitForPolicyUpdate(projectAdminClient, projName, "get", expapi.Resource("horizontalpodautoscalers"), true); err != nil {
t.Fatalf("unexpected error waiting for policy update: %v", err)
}
legacyAutoscalers := legacyExtensionsAutoscaling{
projectAdminKubeClient.Autoscaling().HorizontalPodAutoscalers(projName),
projectAdminKubeClient.AutoscalingClient.RESTClient,
projName,
}
// make sure extensions API objects cannot be listed or created
if _, err := legacyAutoscalers.List(kapi.ListOptions{}); !errors.IsNotFound(err) {
t.Fatalf("expected NotFound error listing HPA, got %v", err)
}
if _, err := legacyAutoscalers.Create(&autoscaling.HorizontalPodAutoscaler{}); !errors.IsNotFound(err) {
t.Fatalf("expected NotFound error creating HPA, got %v", err)
}
if _, err := projectAdminKubeClient14.Extensions().Jobs(projName).List(kapi.ListOptions{}); !errors.IsNotFound(err) {
t.Fatalf("expected NotFound error listing jobs, got %v", err)
}
if _, err := projectAdminKubeClient14.Extensions().Jobs(projName).Create(&extensions_v1beta1.Job{}); !errors.IsNotFound(err) {
t.Fatalf("expected NotFound error creating job, got %v", err)
}
// Delete the containing project
if err := testutil.DeleteAndWaitForNamespaceTermination(clusterAdminKubeClient, projName); err != nil {
t.Fatalf("unexpected error: %#v", err)
}
}