func TestUnschedulableNodes(t *testing.T) {
framework.DeleteAllEtcdKeys()
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("Error in bringing up the master: %v", err)
}
restClient := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
schedulerConfigFactory := factory.NewConfigFactory(restClient, api.DefaultSchedulerName, api.DefaultHardPodAffinitySymmetricWeight, api.DefaultFailureDomains)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
t.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName})
eventBroadcaster.StartRecordingToSink(restClient.Events(""))
scheduler.New(schedulerConfig).Run()
defer close(schedulerConfig.StopEverything)
DoTestUnschedulableNodes(t, restClient, schedulerConfigFactory.NodeLister.Store)
}
func TestUnschedulableNodes(t *testing.T) {
framework.DeleteAllEtcdKeys()
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
m = master.New(masterConfig)
restClient := client.NewOrDie(&client.Config{Host: s.URL, GroupVersion: testapi.Default.GroupVersion()})
schedulerConfigFactory := factory.NewConfigFactory(restClient, nil)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
t.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
eventBroadcaster.StartRecordingToSink(restClient.Events(""))
scheduler.New(schedulerConfig).Run()
defer close(schedulerConfig.StopEverything)
DoTestUnschedulableNodes(t, restClient, schedulerConfigFactory.NodeLister.Store)
}
// TestReadOnlyAuthorization tests that authorization can be controlled
// by namespace.
func TestReadOnlyAuthorization(t *testing.T) {
framework.DeleteAllEtcdKeys()
// This file has alice and bob in it.
// Set up a master
a := newAuthorizerWithContents(t, `{"readonly": true}`)
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
masterConfig.Authenticator = getTestTokenAuth()
masterConfig.Authorizer = a
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("error in bringing up the master: %v", err)
}
transport := http.DefaultTransport
requests := []struct {
verb string
URL string
body string
statusCodes map[int]bool // allowed status codes.
}{
{"POST", path("pods", "", ""), aPod, code403},
{"GET", path("pods", "", ""), "", code200},
{"GET", path("pods", api.NamespaceDefault, "a"), "", code404},
}
for _, r := range requests {
token := BobToken
bodyBytes := bytes.NewReader([]byte(r.body))
req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
func() {
resp, err := transport.RoundTrip(req)
defer resp.Body.Close()
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
if _, ok := r.statusCodes[resp.StatusCode]; !ok {
t.Logf("case %v", r)
t.Errorf("Expected status one of %v, but got %v", r.statusCodes, resp.StatusCode)
b, _ := ioutil.ReadAll(resp.Body)
t.Errorf("Body: %v", string(b))
}
}()
}
}
// TestUnknownUserIsUnauthorized tests that a user who is unknown
// to the authentication system get status code "Unauthorized".
// An authorization module is installed in this scenario for integration
// test purposes, but requests aren't expected to reach it.
func TestUnknownUserIsUnauthorized(t *testing.T) {
framework.DeleteAllEtcdKeys()
// This file has alice and bob in it.
// Set up a master
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
storageDestinations := master.NewStorageDestinations()
storageDestinations.AddAPIGroup("", etcdStorage)
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
m = master.New(&master.Config{
StorageDestinations: storageDestinations,
KubeletClient: client.FakeKubeletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authenticator: getTestTokenAuth(),
Authorizer: allowAliceAuthorizer{},
AdmissionControl: admit.NewAlwaysAdmit(),
StorageVersions: map[string]string{"": testapi.Default.Version()},
})
transport := http.DefaultTransport
for _, r := range getTestRequests() {
token := UnknownToken
bodyBytes := bytes.NewReader([]byte(r.body))
req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
func() {
resp, err := transport.RoundTrip(req)
defer resp.Body.Close()
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
// Expect all of unauthenticated user's request to be "Unauthorized"
if resp.StatusCode != http.StatusUnauthorized {
t.Logf("case %v", r)
t.Errorf("Expected status %v, but got %v", http.StatusUnauthorized, resp.StatusCode)
b, _ := ioutil.ReadAll(resp.Body)
t.Errorf("Body: %v", string(b))
}
}()
}
}
// TestSecrets tests apiserver-side behavior of creation of secret objects and their use by pods.
func TestSecrets(t *testing.T) {
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
m = master.New(&master.Config{
DatabaseStorage: etcdStorage,
KubeletClient: client.FakeKubeletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
StorageVersions: map[string]string{"": testapi.Default.Version()},
})
framework.DeleteAllEtcdKeys()
client := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Default.Version()})
DoTestSecrets(t, client, testapi.Default.Version())
}
func setup(t *testing.T) (*garbagecollector.GarbageCollector, clientset.Interface) {
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// TODO: close the http server
masterConfig := framework.NewIntegrationTestMasterConfig()
masterConfig.EnableCoreControllers = false
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("Error in bringing up the master: %v", err)
}
framework.DeleteAllEtcdKeys()
clientSet, err := clientset.NewForConfig(&restclient.Config{Host: s.URL})
if err != nil {
t.Fatalf("Error in create clientset: %v", err)
}
groupVersionResources, err := clientSet.Discovery().ServerPreferredResources()
if err != nil {
t.Fatalf("Failed to get supported resources from server: %v", err)
}
clientPool := dynamic.NewClientPool(&restclient.Config{Host: s.URL}, dynamic.LegacyAPIPathResolverFunc)
gc, err := garbagecollector.NewGarbageCollector(clientPool, groupVersionResources)
if err != nil {
t.Fatalf("Failed to create garbage collector")
}
return gc, clientSet
}
// mustSetupScheduler starts the following components:
// - k8s api server (a.k.a. master)
// - scheduler
// It returns scheduler config factory and destroyFunc which should be used to
// remove resources after finished.
// Notes on rate limiter:
// - The BindPodsRateLimiter is nil, meaning no rate limits.
// - client rate limit is set to 5000.
func mustSetupScheduler() (schedulerConfigFactory *factory.ConfigFactory, destroyFunc func()) {
framework.DeleteAllEtcdKeys()
var m *master.Master
masterConfig := framework.NewIntegrationTestMasterConfig()
m = master.New(masterConfig)
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
c := client.NewOrDie(&client.Config{
Host: s.URL,
GroupVersion: testapi.Default.GroupVersion(),
QPS: 5000.0,
Burst: 5000,
})
schedulerConfigFactory = factory.NewConfigFactory(c, nil, api.DefaultSchedulerName)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
panic("Couldn't create scheduler config")
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
eventBroadcaster.StartRecordingToSink(c.Events(""))
scheduler.New(schedulerConfig).Run()
destroyFunc = func() {
glog.Infof("destroying")
close(schedulerConfig.StopEverything)
s.Close()
glog.Infof("destroyed")
}
return
}
func TestUnschedulableNodes(t *testing.T) {
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
t.Fatalf("Couldn't create etcd storage: %v", err)
}
expEtcdStorage, err := framework.NewExtensionsEtcdStorage(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
storageDestinations := master.NewStorageDestinations()
storageDestinations.AddAPIGroup("", etcdStorage)
storageDestinations.AddAPIGroup("extensions", expEtcdStorage)
storageVersions := make(map[string]string)
storageVersions[""] = testapi.Default.Version()
storageVersions["extensions"] = testapi.Extensions.GroupAndVersion()
framework.DeleteAllEtcdKeys()
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
m = master.New(&master.Config{
StorageDestinations: storageDestinations,
KubeletClient: client.FakeKubeletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
StorageVersions: storageVersions,
PublicAddress: net.ParseIP("192.168.10.4"),
})
restClient := client.NewOrDie(&client.Config{Host: s.URL, GroupVersion: testapi.Default.GroupVersion()})
schedulerConfigFactory := factory.NewConfigFactory(restClient, nil)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
t.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
eventBroadcaster.StartRecordingToSink(restClient.Events(""))
scheduler.New(schedulerConfig).Run()
defer close(schedulerConfig.StopEverything)
DoTestUnschedulableNodes(t, restClient, schedulerConfigFactory.NodeLister.Store)
}
func TestSelfLinkOnNamespace(t *testing.T) {
_, s := framework.RunAMaster(t)
defer s.Close()
framework.DeleteAllEtcdKeys()
c := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
runSelfLinkTestOnNamespace(t, c, api.NamespaceDefault)
runSelfLinkTestOnNamespace(t, c, "other-namespace")
}
// TestBobIsForbidden tests that a user who is known to
// the authentication system but not authorized to do any actions
// should receive "Forbidden".
func TestBobIsForbidden(t *testing.T) {
framework.DeleteAllEtcdKeys()
// This file has alice and bob in it.
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
m = master.New(&master.Config{
DatabaseStorage: etcdStorage,
KubeletClient: client.FakeKubeletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authenticator: getTestTokenAuth(),
Authorizer: allowAliceAuthorizer{},
AdmissionControl: admit.NewAlwaysAdmit(),
StorageVersions: map[string]string{"": testapi.Default.Version()},
})
transport := http.DefaultTransport
for _, r := range getTestRequests() {
token := BobToken
bodyBytes := bytes.NewReader([]byte(r.body))
req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
func() {
resp, err := transport.RoundTrip(req)
defer resp.Body.Close()
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
// Expect all of bob's actions to return Forbidden
if resp.StatusCode != http.StatusForbidden {
t.Logf("case %v", r)
t.Errorf("Expected not status Forbidden, but got %s", resp.Status)
}
}()
}
}
func TestSelfLinkOnNamespace(t *testing.T) {
// TODO: Limit the test to a single non-default namespace and clean this up at the end.
framework.DeleteAllEtcdKeys()
_, s := framework.RunAMaster(t)
defer s.Close()
c := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
runSelfLinkTestOnNamespace(t, c, api.NamespaceDefault)
runSelfLinkTestOnNamespace(t, c, "other-namespace")
}
func TestAuthModeAlwaysDeny(t *testing.T) {
framework.DeleteAllEtcdKeys()
// Set up a master
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
storageDestinations := master.NewStorageDestinations()
storageDestinations.AddAPIGroup("", etcdStorage)
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
m = master.New(&master.Config{
StorageDestinations: storageDestinations,
KubeletClient: client.FakeKubeletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysDenyAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
StorageVersions: map[string]string{"": testapi.Default.Version()},
})
transport := http.DefaultTransport
for _, r := range getTestRequests() {
bodyBytes := bytes.NewReader([]byte(r.body))
req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes)
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
func() {
resp, err := transport.RoundTrip(req)
defer resp.Body.Close()
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
if resp.StatusCode != http.StatusForbidden {
t.Logf("case %v", r)
t.Errorf("Expected status Forbidden but got status %v", resp.Status)
}
}()
}
}
func TestPersistentVolumeDeleter(t *testing.T) {
// TODO: Limit the test to a single non-default namespace and clean this up at the end.
framework.DeleteAllEtcdKeys()
glog.V(2).Infof("TestPersistentVolumeDeleter started")
_, s := framework.RunAMaster(t)
defer s.Close()
testClient, ctrl, watchPV, watchPVC := createClients(t, s)
defer watchPV.Stop()
defer watchPVC.Stop()
ctrl.Run()
defer ctrl.Stop()
// This PV will be claimed, released, and deleted.
pv := createPV("fake-pv-deleter", "/tmp/foo", "10G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, api.PersistentVolumeReclaimDelete)
pvc := createPVC("fake-pvc-deleter", "5G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce})
_, err := testClient.PersistentVolumes().Create(pv)
if err != nil {
t.Errorf("Failed to create PersistentVolume: %v", err)
}
glog.V(2).Infof("TestPersistentVolumeDeleter pv created")
_, err = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc)
if err != nil {
t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
}
glog.V(2).Infof("TestPersistentVolumeDeleter pvc created")
waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeBound)
glog.V(2).Infof("TestPersistentVolumeDeleter pv bound")
waitForPersistentVolumeClaimPhase(testClient, pvc.Name, watchPVC, api.ClaimBound)
glog.V(2).Infof("TestPersistentVolumeDeleter pvc bound")
// deleting a claim releases the volume, after which it can be recycled
if err := testClient.PersistentVolumeClaims(api.NamespaceDefault).Delete(pvc.Name, nil); err != nil {
t.Errorf("error deleting claim %s", pvc.Name)
}
glog.V(2).Infof("TestPersistentVolumeDeleter pvc deleted")
waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeReleased)
glog.V(2).Infof("TestPersistentVolumeDeleter pv released")
for {
event := <-watchPV.ResultChan()
if event.Type == watch.Deleted {
break
}
}
glog.V(2).Infof("TestPersistentVolumeDeleter pv deleted")
}
// TestSecrets tests apiserver-side behavior of creation of secret objects and their use by pods.
func TestSecrets(t *testing.T) {
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
m = master.New(masterConfig)
framework.DeleteAllEtcdKeys()
client := client.NewOrDie(&client.Config{Host: s.URL, GroupVersion: testapi.Default.GroupVersion()})
DoTestSecrets(t, client, testapi.Default.Version())
}
func BenchmarkScheduling(b *testing.B) {
framework.DeleteAllEtcdKeys()
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
m = master.New(masterConfig)
c := client.NewOrDie(&client.Config{
Host: s.URL,
GroupVersion: testapi.Default.GroupVersion(),
QPS: 5000.0,
Burst: 5000,
})
schedulerConfigFactory := factory.NewConfigFactory(c, nil)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
b.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
eventBroadcaster.StartRecordingToSink(c.Events(""))
scheduler.New(schedulerConfig).Run()
defer close(schedulerConfig.StopEverything)
makeNNodes(c, 1000)
N := b.N
b.ResetTimer()
makeNPods(c, N)
for {
objs := schedulerConfigFactory.ScheduledPodLister.Store.List()
if len(objs) >= N {
fmt.Printf("%v pods scheduled.\n", len(objs))
/* // To prove that this actually works:
for _, o := range objs {
fmt.Printf("%s\n", o.(*api.Pod).Spec.NodeName)
}
*/
break
}
time.Sleep(time.Millisecond)
}
b.StopTimer()
}
// TestUnknownUserIsUnauthorized tests that a user who is unknown
// to the authentication system get status code "Unauthorized".
// An authorization module is installed in this scenario for integration
// test purposes, but requests aren't expected to reach it.
func TestUnknownUserIsUnauthorized(t *testing.T) {
framework.DeleteAllEtcdKeys()
// This file has alice and bob in it.
// Set up a master
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// TODO: Uncomment when fix #19254
// defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
masterConfig.Authenticator = getTestTokenAuth()
masterConfig.Authorizer = allowAliceAuthorizer{}
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("error in bringing up the master: %v", err)
}
transport := http.DefaultTransport
for _, r := range getTestRequests() {
token := UnknownToken
bodyBytes := bytes.NewReader([]byte(r.body))
req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
func() {
resp, err := transport.RoundTrip(req)
defer resp.Body.Close()
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
// Expect all of unauthenticated user's request to be "Unauthorized"
if resp.StatusCode != http.StatusUnauthorized {
t.Logf("case %v", r)
t.Errorf("Expected status %v, but got %v", http.StatusUnauthorized, resp.StatusCode)
b, _ := ioutil.ReadAll(resp.Body)
t.Errorf("Body: %v", string(b))
}
}()
}
}
func TestPersistentVolumeRecycler(t *testing.T) {
glog.V(2).Infof("TestPersistentVolumeRecycler started")
_, s := framework.RunAMaster(t)
defer s.Close()
framework.DeleteAllEtcdKeys()
testClient, ctrl, watchPV, watchPVC := createClients(t, s)
defer watchPV.Stop()
defer watchPVC.Stop()
ctrl.Run()
defer ctrl.Stop()
// This PV will be claimed, released, and recycled.
pv := createPV("fake-pv-recycler", "/tmp/foo", "10G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, api.PersistentVolumeReclaimRecycle)
pvc := createPVC("fake-pvc-recycler", "5G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce})
_, err := testClient.PersistentVolumes().Create(pv)
if err != nil {
t.Errorf("Failed to create PersistentVolume: %v", err)
}
glog.V(2).Infof("TestPersistentVolumeRecycler pvc created")
_, err = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc)
if err != nil {
t.Errorf("Failed to create PersistentVolumeClaim: %v", err)
}
glog.V(2).Infof("TestPersistentVolumeRecycler pvc created")
// wait until the controller pairs the volume and claim
waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeBound)
glog.V(2).Infof("TestPersistentVolumeRecycler pv bound")
waitForPersistentVolumeClaimPhase(testClient, pvc.Name, watchPVC, api.ClaimBound)
glog.V(2).Infof("TestPersistentVolumeRecycler pvc bound")
// deleting a claim releases the volume, after which it can be recycled
if err := testClient.PersistentVolumeClaims(api.NamespaceDefault).Delete(pvc.Name, nil); err != nil {
t.Errorf("error deleting claim %s", pvc.Name)
}
glog.V(2).Infof("TestPersistentVolumeRecycler pvc deleted")
waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeReleased)
glog.V(2).Infof("TestPersistentVolumeRecycler pv released")
waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeAvailable)
glog.V(2).Infof("TestPersistentVolumeRecycler pv available")
}
// TestSecrets tests apiserver-side behavior of creation of secret objects and their use by pods.
func TestSecrets(t *testing.T) {
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("Error in bringing up the master: %v", err)
}
framework.DeleteAllEtcdKeys()
client := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
DoTestSecrets(t, client)
}
// TestBobIsForbidden tests that a user who is known to
// the authentication system but not authorized to do any actions
// should receive "Forbidden".
func TestBobIsForbidden(t *testing.T) {
framework.DeleteAllEtcdKeys()
// This file has alice and bob in it.
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// TODO: Uncomment when fix #19254
// defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
masterConfig.Authenticator = getTestTokenAuth()
masterConfig.Authorizer = allowAliceAuthorizer{}
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("error in bringing up the master: %v", err)
}
transport := http.DefaultTransport
for _, r := range getTestRequests() {
token := BobToken
bodyBytes := bytes.NewReader([]byte(r.body))
req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
func() {
resp, err := transport.RoundTrip(req)
defer resp.Body.Close()
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
// Expect all of bob's actions to return Forbidden
if resp.StatusCode != http.StatusForbidden {
t.Logf("case %v", r)
t.Errorf("Expected not status Forbidden, but got %s", resp.Status)
}
}()
}
}
func init() {
q := flag.Int("bench-quiet", 3, "Quietness, don't glog severities <= given level during the benchmark.")
pods := flag.Int("bench-pods", -1, "Number of pods for the benchmark. If unspecified, uses b.N.")
workers := flag.Int("bench-workers", -1, "Number workers for the benchmark. If unspecified, uses tasks.")
tasks := flag.Int("bench-tasks", -1, "Number of tasks for the benchmark. These tasks are sharded across the workers. If unspecified, uses b.N.")
flag.Parse()
// Unfortunately this v level goes in the opposite direction as stderrthreshold.
flag.Set("v", fmt.Sprintf("%d", *q))
// We need quiet logs to parse benchmark results, which includes Errorf.
flag.Set("logtostderr", "false")
flag.Set("stderrthreshold", fmt.Sprintf("%d", Glog_fatalf-*q))
Pods = *pods
Workers = *workers
Tasks = *tasks
framework.DeleteAllEtcdKeys()
}
func TestMasterService(t *testing.T) {
// TODO: Limit the test to a single non-default namespace and clean this up at the end.
framework.DeleteAllEtcdKeys()
m, err := master.New(framework.NewIntegrationTestMasterConfig())
if err != nil {
t.Fatalf("Error in bringing up the master: %v", err)
}
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
client := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
err = wait.Poll(time.Second, time.Minute, func() (bool, error) {
svcList, err := client.Services(api.NamespaceDefault).List(api.ListOptions{})
if err != nil {
t.Errorf("unexpected error: %v", err)
return false, nil
}
found := false
for i := range svcList.Items {
if svcList.Items[i].Name == "kubernetes" {
found = true
}
}
if found {
ep, err := client.Endpoints(api.NamespaceDefault).Get("kubernetes")
if err != nil {
return false, nil
}
if countEndpoints(ep) == 0 {
return false, fmt.Errorf("no endpoints for kubernetes service: %v", ep)
}
return true, nil
}
return false, nil
})
if err != nil {
t.Errorf("unexpected error: %v", err)
}
}
func TestPodReadOnlyFilesystem(t *testing.T) {
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// TODO: Uncomment when fix #19254
// defer s.Close()
isReadOnly := true
ns := "pod-readonly-root"
masterConfig := framework.NewIntegrationTestMasterConfig()
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("Error in bringing up the master: %v", err)
}
framework.DeleteAllEtcdKeys()
client := client.NewOrDie(&client.Config{Host: s.URL, ContentConfig: client.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{
Name: "XXX",
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "fake-name",
Image: "fakeimage",
SecurityContext: &api.SecurityContext{
ReadOnlyRootFilesystem: &isReadOnly,
},
},
},
},
}
if _, err := client.Pods(ns).Create(pod); err != nil {
t.Errorf("Failed to create pod: %v", err)
}
deletePodOrErrorf(t, client, ns, pod.Name)
}
func TestUnschedulableNodes(t *testing.T) {
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
t.Fatalf("Couldn't create etcd storage: %v", err)
}
framework.DeleteAllEtcdKeys()
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
m = master.New(&master.Config{
DatabaseStorage: etcdStorage,
KubeletClient: client.FakeKubeletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
StorageVersions: map[string]string{"": testapi.Default.Version()},
})
restClient := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Default.Version()})
schedulerConfigFactory := factory.NewConfigFactory(restClient, nil)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
t.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
eventBroadcaster.StartRecordingToSink(restClient.Events(""))
scheduler.New(schedulerConfig).Run()
defer close(schedulerConfig.StopEverything)
DoTestUnschedulableNodes(t, restClient, schedulerConfigFactory.NodeLister.Store)
}
// TestSecrets tests apiserver-side behavior of creation of secret objects and their use by pods.
func TestSecrets(t *testing.T) {
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expEtcdStorage, err := framework.NewExtensionsEtcdStorage(nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
storageDestinations := master.NewStorageDestinations()
storageDestinations.AddAPIGroup("", etcdStorage)
storageDestinations.AddAPIGroup("extensions", expEtcdStorage)
storageVersions := make(map[string]string)
storageVersions[""] = testapi.Default.Version()
storageVersions["extensions"] = testapi.Extensions.GroupAndVersion()
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
m = master.New(&master.Config{
StorageDestinations: storageDestinations,
KubeletClient: kubeletclient.FakeKubeletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
StorageVersions: storageVersions,
PublicAddress: net.ParseIP("192.168.10.4"),
})
framework.DeleteAllEtcdKeys()
client := client.NewOrDie(&client.Config{Host: s.URL, GroupVersion: testapi.Default.GroupVersion()})
DoTestSecrets(t, client, testapi.Default.Version())
}
func TestAuthModeAlwaysDeny(t *testing.T) {
framework.DeleteAllEtcdKeys()
// Set up a master
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// TODO: Uncomment when fix #19254
// defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
masterConfig.Authorizer = apiserver.NewAlwaysDenyAuthorizer()
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("error in bringing up the master: %v", err)
}
transport := http.DefaultTransport
for _, r := range getTestRequests() {
bodyBytes := bytes.NewReader([]byte(r.body))
req, err := http.NewRequest(r.verb, s.URL+r.URL, bodyBytes)
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
func() {
resp, err := transport.RoundTrip(req)
defer resp.Body.Close()
if err != nil {
t.Logf("case %v", r)
t.Fatalf("unexpected error: %v", err)
}
if resp.StatusCode != http.StatusForbidden {
t.Logf("case %v", r)
t.Errorf("Expected status Forbidden but got status %v", resp.Status)
}
}()
}
}
func TestAPIVersions(t *testing.T) {
_, s := framework.RunAMaster(t)
defer s.Close()
framework.DeleteAllEtcdKeys()
c := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
clientVersion := c.APIVersion().String()
g, err := c.ServerGroups()
if err != nil {
t.Fatalf("Failed to get api versions: %v", err)
}
versions := unversioned.ExtractGroupVersions(g)
// Verify that the server supports the API version used by the client.
for _, version := range versions {
if version == clientVersion {
return
}
}
t.Errorf("Server does not support APIVersion used by client. Server supported APIVersions: '%v', client APIVersion: '%v'", versions, clientVersion)
}
func TestUnschedulableNodes(t *testing.T) {
// TODO: Limit the test to a single non-default namespace and clean this up at the end.
framework.DeleteAllEtcdKeys()
_, s := framework.RunAMaster(t)
defer s.Close()
restClient := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
schedulerConfigFactory := factory.NewConfigFactory(restClient, api.DefaultSchedulerName, api.DefaultHardPodAffinitySymmetricWeight, api.DefaultFailureDomains)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
t.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName})
eventBroadcaster.StartRecordingToSink(restClient.Events(""))
scheduler.New(schedulerConfig).Run()
defer close(schedulerConfig.StopEverything)
DoTestUnschedulableNodes(t, restClient, schedulerConfigFactory.NodeLister.Store)
}
func BenchmarkScheduling(b *testing.B) {
etcdStorage, err := framework.NewEtcdStorage()
if err != nil {
b.Fatalf("Couldn't create etcd storage: %v", err)
}
storageDestinations := master.NewStorageDestinations()
storageDestinations.AddAPIGroup("", etcdStorage)
framework.DeleteAllEtcdKeys()
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
defer s.Close()
m = master.New(&master.Config{
StorageDestinations: storageDestinations,
KubeletClient: client.FakeKubeletClient{},
EnableCoreControllers: true,
EnableLogsSupport: false,
EnableUISupport: false,
EnableIndex: true,
APIPrefix: "/api",
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admit.NewAlwaysAdmit(),
StorageVersions: map[string]string{"": testapi.Default.Version()},
})
c := client.NewOrDie(&client.Config{
Host: s.URL,
Version: testapi.Default.Version(),
QPS: 5000.0,
Burst: 5000,
})
schedulerConfigFactory := factory.NewConfigFactory(c, nil)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
b.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"})
eventBroadcaster.StartRecordingToSink(c.Events(""))
scheduler.New(schedulerConfig).Run()
defer close(schedulerConfig.StopEverything)
makeNNodes(c, 1000)
N := b.N
b.ResetTimer()
makeNPods(c, N)
for {
objs := schedulerConfigFactory.ScheduledPodLister.Store.List()
if len(objs) >= N {
fmt.Printf("%v pods scheduled.\n", len(objs))
/* // To prove that this actually works:
for _, o := range objs {
fmt.Printf("%s\n", o.(*api.Pod).Spec.NodeName)
}
*/
break
}
time.Sleep(time.Millisecond)
}
b.StopTimer()
}
func TestSchedulerExtender(t *testing.T) {
framework.DeleteAllEtcdKeys()
var m *master.Master
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
m.Handler.ServeHTTP(w, req)
}))
// TODO: Uncomment when fix #19254
// defer s.Close()
masterConfig := framework.NewIntegrationTestMasterConfig()
m, err := master.New(masterConfig)
if err != nil {
t.Fatalf("error in bringing up the master: %v", err)
}
restClient := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
extender1 := &Extender{
name: "extender1",
predicates: []fitPredicate{machine_1_2_3_Predicate},
prioritizers: []priorityConfig{{machine_2_Prioritizer, 1}},
}
es1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
extender1.serveHTTP(t, w, req)
}))
// TODO: Uncomment when fix #19254
// defer es1.Close()
extender2 := &Extender{
name: "extender2",
predicates: []fitPredicate{machine_2_3_5_Predicate},
prioritizers: []priorityConfig{{machine_3_Prioritizer, 1}},
}
es2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
extender2.serveHTTP(t, w, req)
}))
// TODO: Uncomment when fix #19254
// defer es2.Close()
policy := schedulerapi.Policy{
ExtenderConfigs: []schedulerapi.ExtenderConfig{
{
URLPrefix: es1.URL,
FilterVerb: filter,
PrioritizeVerb: prioritize,
Weight: 3,
EnableHttps: false,
},
{
URLPrefix: es2.URL,
FilterVerb: filter,
PrioritizeVerb: prioritize,
Weight: 4,
EnableHttps: false,
},
},
}
policy.APIVersion = testapi.Default.GroupVersion().String()
schedulerConfigFactory := factory.NewConfigFactory(restClient, api.DefaultSchedulerName)
schedulerConfig, err := schedulerConfigFactory.CreateFromConfig(policy)
if err != nil {
t.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName})
eventBroadcaster.StartRecordingToSink(restClient.Events(""))
scheduler.New(schedulerConfig).Run()
defer close(schedulerConfig.StopEverything)
DoTestPodScheduling(t, restClient)
}
func TestMultiWatch(t *testing.T) {
// Disable this test as long as it demonstrates a problem.
// TODO: Reenable this test when we get #6059 resolved.
return
const watcherCount = 50
rt.GOMAXPROCS(watcherCount)
framework.DeleteAllEtcdKeys()
defer framework.DeleteAllEtcdKeys()
_, s := framework.RunAMaster(t)
defer s.Close()
ns := api.NamespaceDefault
client := client.NewOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
dummyEvent := func(i int) *api.Event {
name := fmt.Sprintf("unrelated-%v", i)
return &api.Event{
ObjectMeta: api.ObjectMeta{
Name: fmt.Sprintf("%v.%x", name, time.Now().UnixNano()),
Namespace: ns,
},
InvolvedObject: api.ObjectReference{
Name: name,
Namespace: ns,
},
Reason: fmt.Sprintf("unrelated change %v", i),
}
}
type timePair struct {
t time.Time
name string
}
receivedTimes := make(chan timePair, watcherCount*2)
watchesStarted := sync.WaitGroup{}
// make a bunch of pods and watch them
for i := 0; i < watcherCount; i++ {
watchesStarted.Add(1)
name := fmt.Sprintf("multi-watch-%v", i)
got, err := client.Pods(ns).Create(&api.Pod{
ObjectMeta: api.ObjectMeta{
Name: name,
Labels: labels.Set{"watchlabel": name},
},
Spec: api.PodSpec{
Containers: []api.Container{{
Name: "pause",
Image: e2e.GetPauseImageName(client),
}},
},
})
if err != nil {
t.Fatalf("Couldn't make %v: %v", name, err)
}
go func(name, rv string) {
options := api.ListOptions{
LabelSelector: labels.Set{"watchlabel": name}.AsSelector(),
ResourceVersion: rv,
}
w, err := client.Pods(ns).Watch(options)
if err != nil {
panic(fmt.Sprintf("watch error for %v: %v", name, err))
}
defer w.Stop()
watchesStarted.Done()
e, ok := <-w.ResultChan() // should get the update (that we'll do below)
if !ok {
panic(fmt.Sprintf("%v ended early?", name))
}
if e.Type != watch.Modified {
panic(fmt.Sprintf("Got unexpected watch notification:\n%v: %+v %+v", name, e, e.Object))
}
receivedTimes <- timePair{time.Now(), name}
}(name, got.ObjectMeta.ResourceVersion)
}
log.Printf("%v: %v pods made and watchers started", time.Now(), watcherCount)
// wait for watches to start before we start spamming the system with
// objects below, otherwise we'll hit the watch window restriction.
watchesStarted.Wait()
const (
useEventsAsUnrelatedType = false
usePodsAsUnrelatedType = true
)
// make a bunch of unrelated changes in parallel
if useEventsAsUnrelatedType {
const unrelatedCount = 3000
var wg sync.WaitGroup
defer wg.Wait()
changeToMake := make(chan int, unrelatedCount*2)
changeMade := make(chan int, unrelatedCount*2)
go func() {
for i := 0; i < unrelatedCount; i++ {
changeToMake <- i
}
close(changeToMake)
}()
for i := 0; i < 50; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
i, ok := <-changeToMake
if !ok {
return
}
if _, err := client.Events(ns).Create(dummyEvent(i)); err != nil {
panic(fmt.Sprintf("couldn't make an event: %v", err))
}
changeMade <- i
}
}()
}
for i := 0; i < 2000; i++ {
<-changeMade
if (i+1)%50 == 0 {
log.Printf("%v: %v unrelated changes made", time.Now(), i+1)
}
}
}
if usePodsAsUnrelatedType {
const unrelatedCount = 3000
var wg sync.WaitGroup
defer wg.Wait()
changeToMake := make(chan int, unrelatedCount*2)
changeMade := make(chan int, unrelatedCount*2)
go func() {
for i := 0; i < unrelatedCount; i++ {
changeToMake <- i
}
close(changeToMake)
}()
for i := 0; i < 50; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
i, ok := <-changeToMake
if !ok {
return
}
name := fmt.Sprintf("unrelated-%v", i)
_, err := client.Pods(ns).Create(&api.Pod{
ObjectMeta: api.ObjectMeta{
Name: name,
},
Spec: api.PodSpec{
Containers: []api.Container{{
Name: "nothing",
Image: e2e.GetPauseImageName(client),
}},
},
})
if err != nil {
panic(fmt.Sprintf("couldn't make unrelated pod: %v", err))
}
changeMade <- i
}
}()
}
for i := 0; i < 2000; i++ {
<-changeMade
if (i+1)%50 == 0 {
log.Printf("%v: %v unrelated changes made", time.Now(), i+1)
}
}
}
// Now we still have changes being made in parallel, but at least 1000 have been made.
// Make some updates to send down the watches.
sentTimes := make(chan timePair, watcherCount*2)
for i := 0; i < watcherCount; i++ {
go func(i int) {
name := fmt.Sprintf("multi-watch-%v", i)
pod, err := client.Pods(ns).Get(name)
if err != nil {
panic(fmt.Sprintf("Couldn't get %v: %v", name, err))
}
pod.Spec.Containers[0].Image = e2e.GetPauseImageName(client)
sentTimes <- timePair{time.Now(), name}
if _, err := client.Pods(ns).Update(pod); err != nil {
panic(fmt.Sprintf("Couldn't make %v: %v", name, err))
}
}(i)
}
sent := map[string]time.Time{}
for i := 0; i < watcherCount; i++ {
tp := <-sentTimes
sent[tp.name] = tp.t
}
log.Printf("all changes made")
dur := map[string]time.Duration{}
for i := 0; i < watcherCount; i++ {
tp := <-receivedTimes
delta := tp.t.Sub(sent[tp.name])
dur[tp.name] = delta
log.Printf("%v: %v", tp.name, delta)
}
log.Printf("all watches ended")
t.Errorf("durations: %v", dur)
}