func TestGet(t *testing.T) {
expect := validNewPod()
expect.Status.Phase = api.PodRunning
expect.Spec.NodeName = "machine"
fakeEtcdClient, helper := newHelper(t)
key := etcdtest.AddPrefix("/pods/test/foo")
fakeEtcdClient.Data[key] = tools.EtcdResponseWithError{
R: &etcd.Response{
Node: &etcd.Node{
Value: runtime.EncodeOrDie(latest.Codec, expect),
},
},
}
storage := NewStorage(helper, nil).Pod
obj, err := storage.Get(api.WithNamespace(api.NewContext(), "test"), "foo")
pod := obj.(*api.Pod)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if e, a := expect, pod; !api.Semantic.DeepEqual(e, a) {
t.Errorf("Unexpected pod: %s", util.ObjectDiff(e, a))
}
}
// Bind just does a POST binding RPC.
func (b *binder) Bind(binding *api.Binding) error {
glog.V(2).Infof("Attempting to bind %v to %v", binding.Name, binding.Target.Name)
ctx := api.WithNamespace(api.NewContext(), binding.Namespace)
return b.Post().Namespace(api.NamespaceValue(ctx)).Resource("bindings").Body(binding).Do().Error()
// TODO: use Pods interface for binding once clusters are upgraded
// return b.Pods(binding.Namespace).Bind(binding)
}
func (k *deleter) deleteOne(pod *Pod) error {
ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace)
podKey, err := podtask.MakePodKey(ctx, pod.Name)
if err != nil {
return err
}
log.V(2).Infof("pod deleted: %v", podKey)
// order is important here: we want to make sure we have the lock before
// removing the pod from the scheduling queue. this makes the concurrent
// execution of scheduler-error-handling and delete-handling easier to
// reason about.
k.api.Lock()
defer k.api.Unlock()
// prevent the scheduler from attempting to pop this; it's also possible that
// it's concurrently being scheduled (somewhere between pod scheduling and
// binding) - if so, then we'll end up removing it from taskRegistry which
// will abort Bind()ing
k.qr.dequeue(pod.GetUID())
switch task, state := k.api.tasks().ForPod(podKey); state {
case podtask.StateUnknown:
log.V(2).Infof("Could not resolve pod '%s' to task id", podKey)
return noSuchPodErr
// determine if the task has already been launched to mesos, if not then
// cleanup is easier (unregister) since there's no state to sync
case podtask.StatePending:
if !task.Has(podtask.Launched) {
// we've been invoked in between Schedule() and Bind()
if task.HasAcceptedOffer() {
task.Offer.Release()
task.Reset()
task.Set(podtask.Deleted)
//TODO(jdef) probably want better handling here
if err := k.api.tasks().Update(task); err != nil {
return err
}
}
k.api.tasks().Unregister(task)
return nil
}
fallthrough
case podtask.StateRunning:
// signal to watchers that the related pod is going down
task.Set(podtask.Deleted)
if err := k.api.tasks().Update(task); err != nil {
log.Errorf("failed to update task w/ Deleted status: %v", err)
}
return k.api.killTask(task.ID)
default:
log.Infof("cannot kill pod '%s': non-terminal task not found %v", podKey, task.ID)
return noSuchTaskErr
}
}
// this pod may be out of sync with respect to the API server registry:
// this pod | apiserver registry
// -------------|----------------------
// host=.* | 404 ; pod was deleted
// host=.* | 5xx ; failed to sync, try again later?
// host="" | host="" ; perhaps no updates to process?
// host="" | host="..." ; pod has been scheduled and assigned, is there a task assigned? (check TaskIdKey in binding?)
// host="..." | host="" ; pod is no longer scheduled, does it need to be re-queued?
// host="..." | host="..." ; perhaps no updates to process?
//
// TODO(jdef) this needs an integration test
func (s *schedulingPlugin) reconcilePod(oldPod api.Pod) {
log.V(1).Infof("reconcile pod %v", oldPod.Name)
ctx := api.WithNamespace(api.NewDefaultContext(), oldPod.Namespace)
pod, err := s.client.Pods(api.NamespaceValue(ctx)).Get(oldPod.Name)
if err != nil {
if errors.IsNotFound(err) {
// attempt to delete
if err = s.deleter.deleteOne(&Pod{Pod: &oldPod}); err != nil && err != noSuchPodErr && err != noSuchTaskErr {
log.Errorf("failed to delete pod: %v: %v", oldPod.Name, err)
}
} else {
//TODO(jdef) other errors should probably trigger a retry (w/ backoff).
//For now, drop the pod on the floor
log.Warning("aborting reconciliation for pod %v: %v", oldPod.Name, err)
}
return
}
if oldPod.Spec.NodeName != pod.Spec.NodeName {
if pod.Spec.NodeName == "" {
// pod is unscheduled.
// it's possible that we dropped the pod in the scheduler error handler
// because of task misalignment with the pod (task.Has(podtask.Launched) == true)
podKey, err := podtask.MakePodKey(ctx, pod.Name)
if err != nil {
log.Error(err)
return
}
s.api.Lock()
defer s.api.Unlock()
if _, state := s.api.tasks().ForPod(podKey); state != podtask.StateUnknown {
//TODO(jdef) reconcile the task
log.Errorf("task already registered for pod %v", pod.Name)
return
}
now := time.Now()
log.V(3).Infof("reoffering pod %v", podKey)
s.qr.reoffer(&Pod{
Pod: pod,
deadline: &now,
})
} else {
// pod is scheduled.
// not sure how this happened behind our backs. attempt to reconstruct
// at least a partial podtask.T record.
//TODO(jdef) reconcile the task
log.Errorf("pod already scheduled: %v", pod.Name)
}
} else {
//TODO(jdef) for now, ignore the fact that the rest of the spec may be different
//and assume that our knowledge of the pod aligns with that of the apiserver
log.Error("pod reconciliation does not support updates; not yet implemented")
}
}
// createController is a helper function that returns a controller with the updated resource version.
func createController(storage *REST, rc api.ReplicationController, t *testing.T) (api.ReplicationController, error) {
ctx := api.WithNamespace(api.NewContext(), rc.Namespace)
obj, err := storage.Create(ctx, &rc)
if err != nil {
t.Errorf("Failed to create controller, %v", err)
}
newRc := obj.(*api.ReplicationController)
return *newRc, nil
}
// implementation of scheduling plugin's Error func; see plugin/pkg/scheduler
func (k *errorHandler) handleSchedulingError(pod *api.Pod, schedulingErr error) {
if schedulingErr == noSuchPodErr {
log.V(2).Infof("Not rescheduling non-existent pod %v", pod.Name)
return
}
log.Infof("Error scheduling %v: %v; retrying", pod.Name, schedulingErr)
defer util.HandleCrash()
// default upstream scheduler passes pod.Name as binding.PodID
ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace)
podKey, err := podtask.MakePodKey(ctx, pod.Name)
if err != nil {
log.Errorf("Failed to construct pod key, aborting scheduling for pod %v: %v", pod.Name, err)
return
}
k.backoff.GC()
k.api.Lock()
defer k.api.Unlock()
switch task, state := k.api.tasks().ForPod(podKey); state {
case podtask.StateUnknown:
// if we don't have a mapping here any more then someone deleted the pod
log.V(2).Infof("Could not resolve pod to task, aborting pod reschdule: %s", podKey)
return
case podtask.StatePending:
if task.Has(podtask.Launched) {
log.V(2).Infof("Skipping re-scheduling for already-launched pod %v", podKey)
return
}
breakoutEarly := queue.BreakChan(nil)
if schedulingErr == noSuitableOffersErr {
log.V(3).Infof("adding backoff breakout handler for pod %v", podKey)
breakoutEarly = queue.BreakChan(k.api.offers().Listen(podKey, func(offer *mesos.Offer) bool {
k.api.Lock()
defer k.api.Unlock()
switch task, state := k.api.tasks().Get(task.ID); state {
case podtask.StatePending:
return !task.Has(podtask.Launched) && task.AcceptOffer(offer)
default:
// no point in continuing to check for matching offers
return true
}
}))
}
delay := k.backoff.Get(podKey)
log.V(3).Infof("requeuing pod %v with delay %v", podKey, delay)
k.qr.requeue(&Pod{Pod: pod, delay: &delay, notify: breakoutEarly})
default:
log.V(2).Infof("Task is no longer pending, aborting reschedule for pod %v", podKey)
}
}
// Schedule implements the Scheduler interface of QingYuan.
// It returns the selectedMachine's name and error (if there's any).
func (k *qingScheduler) Schedule(pod *api.Pod, unused algorithm.MinionLister) (string, error) {
log.Infof("Try to schedule pod %v\n", pod.Name)
ctx := api.WithNamespace(api.NewDefaultContext(), pod.Namespace)
// default upstream scheduler passes pod.Name as binding.PodID
podKey, err := podtask.MakePodKey(ctx, pod.Name)
if err != nil {
return "", err
}
k.api.Lock()
defer k.api.Unlock()
switch task, state := k.api.tasks().ForPod(podKey); state {
case podtask.StateUnknown:
// There's a bit of a potential race here, a pod could have been yielded() and
// then before we get *here* it could be deleted.
// We use meta to index the pod in the store since that's what k8s reflector does.
podName, err := cache.MetaNamespaceKeyFunc(pod)
if err != nil {
log.Warningf("aborting Schedule, unable to understand pod object %+v", pod)
return "", noSuchPodErr
}
if deleted := k.podUpdates.Poll(podName, queue.DELETE_EVENT); deleted {
// avoid scheduling a pod that's been deleted between yieldPod() and Schedule()
log.Infof("aborting Schedule, pod has been deleted %+v", pod)
return "", noSuchPodErr
}
return k.doSchedule(k.api.tasks().Register(k.api.createPodTask(ctx, pod)))
//TODO(jdef) it's possible that the pod state has diverged from what
//we knew previously, we should probably update the task.Pod state here
//before proceeding with scheduling
case podtask.StatePending:
if pod.UID != task.Pod.UID {
// we're dealing with a brand new pod spec here, so the old one must have been
// deleted -- and so our task store is out of sync w/ respect to reality
//TODO(jdef) reconcile task
return "", fmt.Errorf("task %v spec is out of sync with pod %v spec, aborting schedule", task.ID, pod.Name)
} else if task.Has(podtask.Launched) {
// task has been marked as "launched" but the pod binding creation may have failed in k8s,
// but we're going to let someone else handle it, probably the mesos task error handler
return "", fmt.Errorf("task %s has already been launched, aborting schedule", task.ID)
} else {
return k.doSchedule(task, nil)
}
default:
return "", fmt.Errorf("task %s is not pending, nothing to schedule", task.ID)
}
}
func (t *Tester) TestDeleteGracefulUsesZeroOnNil(existing runtime.Object, expectedGrace int64) {
objectMeta, err := api.ObjectMetaFor(existing)
if err != nil {
t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
}
ctx := api.WithNamespace(t.TestContext(), objectMeta.Namespace)
_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, nil)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); !errors.IsNotFound(err) {
t.Errorf("unexpected error, object should exist: %v", err)
}
}
func (t *Tester) TestCreateIgnoresContextNamespace(valid runtime.Object) {
// Ignore non-empty namespace in context
ctx := api.WithNamespace(api.NewContext(), "not-default2")
// Ideally, we'd get an error back here, but at least verify the namespace wasn't persisted
created, err := t.storage.(rest.Creater).Create(ctx, copyOrDie(valid))
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
createdObjectMeta, err := api.ObjectMetaFor(created)
if err != nil {
t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, created)
}
if createdObjectMeta.Namespace != api.NamespaceNone {
t.Errorf("Expected empty namespace on created object, got '%v'", createdObjectMeta.Namespace)
}
}
func TestRESTCreate(t *testing.T) {
table := []struct {
ctx api.Context
event *api.Event
valid bool
}{
{
ctx: api.NewDefaultContext(),
event: testEvent("foo"),
valid: true,
}, {
ctx: api.NewContext(),
event: testEvent("bar"),
valid: true,
}, {
ctx: api.WithNamespace(api.NewContext(), "nondefault"),
event: testEvent("bazzzz"),
valid: false,
},
}
for _, item := range table {
_, storage := NewTestREST()
c, err := storage.Create(item.ctx, item.event)
if !item.valid {
if err == nil {
ctxNS := api.NamespaceValue(item.ctx)
t.Errorf("unexpected non-error for %v (%v, %v)", item.event.Name, ctxNS, item.event.Namespace)
}
continue
}
if err != nil {
t.Errorf("%v: Unexpected error %v", item.event.Name, err)
continue
}
if !api.HasObjectMetaSystemFieldValues(&item.event.ObjectMeta) {
t.Errorf("storage did not populate object meta field values")
}
if e, a := item.event, c; !reflect.DeepEqual(e, a) {
t.Errorf("diff: %s", util.ObjectDiff(e, a))
}
// Ensure we implement the interface
_ = rest.Watcher(storage)
}
}
func (t *Tester) TestDeleteGracefulHasDefault(existing runtime.Object, expectedGrace int64, wasGracefulFn func() bool) {
objectMeta, err := api.ObjectMetaFor(existing)
if err != nil {
t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
}
ctx := api.WithNamespace(t.TestContext(), objectMeta.Namespace)
_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, &api.DeleteOptions{})
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); err != nil {
t.Errorf("unexpected error, object should exist: %v", err)
}
if !wasGracefulFn() {
t.Errorf("did not gracefully delete resource")
}
}
func (t *Tester) TestDeleteNoGraceful(createFn func() runtime.Object, wasGracefulFn func() bool) {
existing := createFn()
objectMeta, err := api.ObjectMetaFor(existing)
if err != nil {
t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
}
ctx := api.WithNamespace(t.TestContext(), objectMeta.Namespace)
_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(10))
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); !errors.IsNotFound(err) {
t.Errorf("unexpected error, object should not exist: %v", err)
}
if wasGracefulFn() {
t.Errorf("resource should not support graceful delete")
}
}
// TestEtcdGetControllerDifferentNamespace ensures same-name controllers in different namespaces do not clash
func TestEtcdGetControllerDifferentNamespace(t *testing.T) {
storage, fakeClient := newStorage(t)
otherNs := "other"
ctx1 := api.NewDefaultContext()
ctx2 := api.WithNamespace(api.NewContext(), otherNs)
key1, _ := makeControllerKey(ctx1, validController.Name)
key2, _ := makeControllerKey(ctx2, validController.Name)
key1 = etcdtest.AddPrefix(key1)
key2 = etcdtest.AddPrefix(key2)
fakeClient.Set(key1, runtime.EncodeOrDie(latest.Codec, &validController), 0)
otherNsController := validController
otherNsController.Namespace = otherNs
fakeClient.Set(key2, runtime.EncodeOrDie(latest.Codec, &otherNsController), 0)
obj, err := storage.Get(ctx1, validController.Name)
ctrl1, _ := obj.(*api.ReplicationController)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if ctrl1.Name != "foo" {
t.Errorf("Unexpected controller: %#v", ctrl1)
}
if ctrl1.Namespace != "default" {
t.Errorf("Unexpected controller: %#v", ctrl1)
}
obj, err = storage.Get(ctx2, validController.Name)
ctrl2, _ := obj.(*api.ReplicationController)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if ctrl2.Name != "foo" {
t.Errorf("Unexpected controller: %#v", ctrl2)
}
if ctrl2.Namespace != "other" {
t.Errorf("Unexpected controller: %#v", ctrl2)
}
}
func TestEtcdWatchControllersMatch(t *testing.T) {
ctx := api.WithNamespace(api.NewDefaultContext(), validController.Namespace)
storage, fakeClient := newStorage(t)
fakeClient.ExpectNotFoundGet(etcdgeneric.NamespaceKeyRootFunc(ctx, "/registry/pods"))
watching, err := storage.Watch(ctx,
labels.SelectorFromSet(validController.Spec.Selector),
fields.Everything(),
"1",
)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
fakeClient.WaitForWatchCompletion()
// The watcher above is waiting for these Labels, on receiving them it should
// apply the ControllerStatus decorator, which lists pods, causing a query against
// the /registry/pods endpoint of the etcd client.
controller := &api.ReplicationController{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Labels: validController.Spec.Selector,
Namespace: "default",
},
}
controllerBytes, _ := latest.Codec.Encode(controller)
fakeClient.WatchResponse <- &etcd.Response{
Action: "create",
Node: &etcd.Node{
Value: string(controllerBytes),
},
}
select {
case _, ok := <-watching.ResultChan():
if !ok {
t.Errorf("watching channel should be open")
}
case <-time.After(time.Millisecond * 100):
t.Error("unexpected timeout from result channel")
}
watching.Stop()
}
// ConnectResource returns a function that handles a connect request on a rest.Storage object.
func ConnectResource(connecter rest.Connecter, scope RequestScope, admit admission.Interface, connectOptionsKind, restPath string, subpath bool, subpathKey string) restful.RouteFunction {
return func(req *restful.Request, res *restful.Response) {
w := res.ResponseWriter
namespace, name, err := scope.Namer.Name(req)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
ctx := scope.ContextFunc(req)
ctx = api.WithNamespace(ctx, namespace)
opts, err := getRequestOptions(req, scope, connectOptionsKind, subpath, subpathKey)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if admit.Handles(admission.Connect) {
connectRequest := &rest.ConnectRequest{
Name: name,
Options: opts,
ResourcePath: restPath,
}
userInfo, _ := api.UserFrom(ctx)
err = admit.Admit(admission.NewAttributesRecord(connectRequest, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Connect, userInfo))
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
}
handler, err := connecter.Connect(ctx, name, opts)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
handler.ServeHTTP(w, req.Request)
err = handler.RequestError()
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
}
}
// TestEtcdGetDifferentNamespace ensures same-name pods in different namespaces do not clash
func TestEtcdGetDifferentNamespace(t *testing.T) {
registry, _, _, fakeClient, _ := newStorage(t)
ctx1 := api.NewDefaultContext()
ctx2 := api.WithNamespace(api.NewContext(), "other")
key1, _ := registry.KeyFunc(ctx1, "foo")
key2, _ := registry.KeyFunc(ctx2, "foo")
key1 = etcdtest.AddPrefix(key1)
key2 = etcdtest.AddPrefix(key2)
fakeClient.Set(key1, runtime.EncodeOrDie(latest.Codec, &api.Pod{ObjectMeta: api.ObjectMeta{Namespace: "default", Name: "foo"}}), 0)
fakeClient.Set(key2, runtime.EncodeOrDie(latest.Codec, &api.Pod{ObjectMeta: api.ObjectMeta{Namespace: "other", Name: "foo"}}), 0)
obj, err := registry.Get(ctx1, "foo")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
pod1 := obj.(*api.Pod)
if pod1.Name != "foo" {
t.Errorf("Unexpected pod: %#v", pod1)
}
if pod1.Namespace != "default" {
t.Errorf("Unexpected pod: %#v", pod1)
}
obj, err = registry.Get(ctx2, "foo")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
pod2 := obj.(*api.Pod)
if pod2.Name != "foo" {
t.Errorf("Unexpected pod: %#v", pod2)
}
if pod2.Namespace != "other" {
t.Errorf("Unexpected pod: %#v", pod2)
}
}
// TestEtcdGetServiceDifferentNamespace ensures same-name services in different namespaces do not clash
func TestEtcdGetServiceDifferentNamespace(t *testing.T) {
fakeClient := tools.NewFakeEtcdClient(t)
registry := NewTestEtcdRegistry(fakeClient)
ctx1 := api.NewDefaultContext()
ctx2 := api.WithNamespace(api.NewContext(), "other")
key1, _ := makeServiceKey(ctx1, "foo")
key2, _ := makeServiceKey(ctx2, "foo")
key1 = etcdtest.AddPrefix(key1)
key2 = etcdtest.AddPrefix(key2)
fakeClient.Set(key1, runtime.EncodeOrDie(latest.Codec, &api.Service{ObjectMeta: api.ObjectMeta{Namespace: "default", Name: "foo"}}), 0)
fakeClient.Set(key2, runtime.EncodeOrDie(latest.Codec, &api.Service{ObjectMeta: api.ObjectMeta{Namespace: "other", Name: "foo"}}), 0)
service1, err := registry.GetService(ctx1, "foo")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if service1.Name != "foo" {
t.Errorf("Unexpected service: %#v", service1)
}
if service1.Namespace != "default" {
t.Errorf("Unexpected service: %#v", service1)
}
service2, err := registry.GetService(ctx2, "foo")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if service2.Name != "foo" {
t.Errorf("Unexpected service: %#v", service2)
}
if service2.Namespace != "other" {
t.Errorf("Unexpected service: %#v", service2)
}
}
// Benchmark pod listing by waiting on `Tasks` listers to list `Pods` pods via `Workers`.
func BenchmarkPodListEtcd(b *testing.B) {
b.StopTimer()
m := framework.NewMasterComponents(&framework.Config{nil, true, false, 250.0, 500})
defer m.Stop(true, true)
numPods, numTasks, iter := getPods(b.N), getTasks(b.N), getIterations(b.N)
podsPerNode := numPods / numTasks
if podsPerNode < 1 {
podsPerNode = 1
}
startPodsOnNodes(numPods, numTasks, m.RestClient)
// Stop the rc manager so it doesn't steal resources
m.Stop(false, true)
glog.Infof("Starting benchmark: b.N %d, pods %d, workers %d, podsPerNode %d",
b.N, numPods, numTasks, podsPerNode)
ctx := api.WithNamespace(api.NewContext(), framework.TestNS)
key := etcdgeneric.NamespaceKeyRootFunc(ctx, fmt.Sprintf("%s/pods", etcdtest.PathPrefix()))
b.StartTimer()
for i := 0; i < iter; i++ {
framework.RunParallel(func(id int) error {
now := time.Now()
defer func() {
glog.V(3).Infof("Worker %d: listing pods took %v", id, time.Since(now))
}()
if response, err := m.EtcdHelper.Client.Get(key, true, true); err != nil {
return err
} else if len(response.Node.Nodes) < podsPerNode {
glog.Fatalf("List retrieved %d pods, which is less than %d", len(response.Node.Nodes), podsPerNode)
}
return nil
}, numTasks, Workers)
}
b.StopTimer()
}
// implements binding.Registry, launches the pod-associated-task in mesos
func (b *binder) Bind(binding *api.Binding) error {
ctx := api.WithNamespace(api.NewContext(), binding.Namespace)
// default upstream scheduler passes pod.Name as binding.Name
podKey, err := podtask.MakePodKey(ctx, binding.Name)
if err != nil {
return err
}
b.api.Lock()
defer b.api.Unlock()
switch task, state := b.api.tasks().ForPod(podKey); state {
case podtask.StatePending:
return b.bind(ctx, binding, task)
default:
// in this case it's likely that the pod has been deleted between Schedule
// and Bind calls
log.Infof("No pending task for pod %s", podKey)
return noSuchPodErr //TODO(jdef) this error is somewhat misleading since the task could be running?!
}
}
func TestGet(t *testing.T) {
expect := validNewResourceQuota()
fakeEtcdClient, helper := newHelper(t)
storage, _ := NewStorage(helper)
key := "/resourcequotas/test/foo"
key = etcdtest.AddPrefix(key)
fakeEtcdClient.Data[key] = tools.EtcdResponseWithError{
R: &etcd.Response{
Node: &etcd.Node{
Value: runtime.EncodeOrDie(latest.Codec, expect),
},
},
}
obj, err := storage.Get(api.WithNamespace(api.NewContext(), "test"), "foo")
resourcequota := obj.(*api.ResourceQuota)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if e, a := expect, resourcequota; !api.Semantic.DeepEqual(e, a) {
t.Errorf("Unexpected resourcequota: %s", util.ObjectDiff(e, a))
}
}
func TestEtcdControllerValidatesNamespaceOnUpdate(t *testing.T) {
storage, _ := newStorage(t)
ns := "newnamespace"
// The update should fail if the namespace on the controller is set to something
// other than the namespace on the given context, even if the namespace on the
// controller is valid.
updateController, err := createController(storage, validController, t)
newNamespaceController := validController
newNamespaceController.Namespace = ns
_, err = createController(storage, newNamespaceController, t)
c, updated, err := storage.Update(api.WithNamespace(api.NewContext(), ns), &updateController)
if c != nil || updated {
t.Errorf("Expected nil object and not created")
}
// TODO: Be more paranoid about the type of error and make sure it has the substring
// "namespace" in it, once #5684 is fixed. Ideally this would be a NewBadRequest.
if err == nil {
t.Errorf("Expected an error, but we didn't get one")
}
}
// getResourceHandler is an HTTP handler function for get requests. It delegates to the
// passed-in getterFunc to perform the actual get.
func getResourceHandler(scope RequestScope, getter getterFunc) restful.RouteFunction {
return func(req *restful.Request, res *restful.Response) {
w := res.ResponseWriter
namespace, name, err := scope.Namer.Name(req)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
ctx := scope.ContextFunc(req)
ctx = api.WithNamespace(ctx, namespace)
result, err := getter(ctx, name, req)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if err := setSelfLink(result, req, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
write(http.StatusOK, scope.APIVersion, scope.Codec, result, w, req.Request)
}
}
// RunOnce verifies the state of the port allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) RunOnce() error {
// TODO: (per smarterclayton) if Get() or ListServices() is a weak consistency read,
// or if they are executed against different leaders,
// the ordering guarantee required to ensure no port is allocated twice is violated.
// ListServices must return a ResourceVersion higher than the etcd index Get triggers,
// and the release code must not release services that have had ports allocated but not yet been created
// See #8295
// If etcd server is not running we should wait for some time and fail only then. This is particularly
// important when we start apiserver and etcd at the same time.
var latest *api.RangeAllocation
var err error
for i := 0; i < 10; i++ {
if latest, err = c.alloc.Get(); err != nil {
time.Sleep(time.Second)
} else {
break
}
}
if err != nil {
return fmt.Errorf("unable to refresh the port block: %v", err)
}
ctx := api.WithNamespace(api.NewDefaultContext(), api.NamespaceAll)
list, err := c.registry.ListServices(ctx)
if err != nil {
return fmt.Errorf("unable to refresh the port block: %v", err)
}
r := portallocator.NewPortAllocator(c.portRange)
for i := range list.Items {
svc := &list.Items[i]
ports := service.CollectServiceNodePorts(svc)
if len(ports) == 0 {
continue
}
for _, port := range ports {
switch err := r.Allocate(port); err {
case nil:
case portallocator.ErrAllocated:
// TODO: send event
// port is broken, reallocate
util.HandleError(fmt.Errorf("the port %d for service %s/%s was assigned to multiple services; please recreate", port, svc.Name, svc.Namespace))
case portallocator.ErrNotInRange:
// TODO: send event
// port is broken, reallocate
util.HandleError(fmt.Errorf("the port %d for service %s/%s is not within the port range %v; please recreate", port, svc.Name, svc.Namespace, c.portRange))
case portallocator.ErrFull:
// TODO: send event
return fmt.Errorf("the port range %v is full; you must widen the port range in order to create new services", c.portRange)
default:
return fmt.Errorf("unable to allocate port %d for service %s/%s due to an unknown error, exiting: %v", port, svc.Name, svc.Namespace, err)
}
}
}
err = r.Snapshot(latest)
if err != nil {
return fmt.Errorf("unable to persist the updated port allocations: %v", err)
}
if err := c.alloc.CreateOrUpdate(latest); err != nil {
return fmt.Errorf("unable to persist the updated port allocations: %v", err)
}
return nil
}
// RunOnce verifies the state of the cluster IP allocations and returns an error if an unrecoverable problem occurs.
func (c *Repair) RunOnce() error {
// TODO: (per smarterclayton) if Get() or ListServices() is a weak consistency read,
// or if they are executed against different leaders,
// the ordering guarantee required to ensure no IP is allocated twice is violated.
// ListServices must return a ResourceVersion higher than the etcd index Get triggers,
// and the release code must not release services that have had IPs allocated but not yet been created
// See #8295
// If etcd server is not running we should wait for some time and fail only then. This is particularly
// important when we start apiserver and etcd at the same time.
var latest *api.RangeAllocation
var err error
for i := 0; i < 10; i++ {
if latest, err = c.alloc.Get(); err != nil {
time.Sleep(time.Second)
} else {
break
}
}
if err != nil {
return fmt.Errorf("unable to refresh the service IP block: %v", err)
}
ctx := api.WithNamespace(api.NewDefaultContext(), api.NamespaceAll)
list, err := c.registry.ListServices(ctx)
if err != nil {
return fmt.Errorf("unable to refresh the service IP block: %v", err)
}
r := ipallocator.NewCIDRRange(c.network)
for _, svc := range list.Items {
if !api.IsServiceIPSet(&svc) {
continue
}
ip := net.ParseIP(svc.Spec.ClusterIP)
if ip == nil {
// cluster IP is broken, reallocate
util.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not a valid IP; please recreate", svc.Spec.ClusterIP, svc.Name, svc.Namespace))
continue
}
switch err := r.Allocate(ip); err {
case nil:
case ipallocator.ErrAllocated:
// TODO: send event
// cluster IP is broken, reallocate
util.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s was assigned to multiple services; please recreate", ip, svc.Name, svc.Namespace))
case ipallocator.ErrNotInRange:
// TODO: send event
// cluster IP is broken, reallocate
util.HandleError(fmt.Errorf("the cluster IP %s for service %s/%s is not within the service CIDR %s; please recreate", ip, svc.Name, svc.Namespace, c.network))
case ipallocator.ErrFull:
// TODO: send event
return fmt.Errorf("the service CIDR %s is full; you must widen the CIDR in order to create new services")
default:
return fmt.Errorf("unable to allocate cluster IP %s for service %s/%s due to an unknown error, exiting: %v", ip, svc.Name, svc.Namespace, err)
}
}
err = r.Snapshot(latest)
if err != nil {
return fmt.Errorf("unable to persist the updated service IP allocations: %v", err)
}
if err := c.alloc.CreateOrUpdate(latest); err != nil {
return fmt.Errorf("unable to persist the updated service IP allocations: %v", err)
}
return nil
}
// DeleteResource returns a function that will handle a resource deletion
func DeleteResource(r rest.GracefulDeleter, checkBody bool, scope RequestScope, admit admission.Interface) restful.RouteFunction {
return func(req *restful.Request, res *restful.Response) {
w := res.ResponseWriter
// TODO: we either want to remove timeout or document it (if we document, move timeout out of this function and declare it in api_installer)
timeout := parseTimeout(req.Request.URL.Query().Get("timeout"))
namespace, name, err := scope.Namer.Name(req)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
ctx := scope.ContextFunc(req)
ctx = api.WithNamespace(ctx, namespace)
options := &api.DeleteOptions{}
if checkBody {
body, err := readBody(req.Request)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if len(body) > 0 {
if err := scope.Codec.DecodeInto(body, options); err != nil {
errorJSON(err, scope.Codec, w)
return
}
}
}
if admit.Handles(admission.Delete) {
userInfo, _ := api.UserFrom(ctx)
err = admit.Admit(admission.NewAttributesRecord(nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Delete, userInfo))
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
}
result, err := finishRequest(timeout, func() (runtime.Object, error) {
return r.Delete(ctx, name, options)
})
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
// if the rest.Deleter returns a nil object, fill out a status. Callers may return a valid
// object with the response.
if result == nil {
result = &api.Status{
Status: api.StatusSuccess,
Code: http.StatusOK,
Details: &api.StatusDetails{
Name: name,
Kind: scope.Kind,
},
}
} else {
// when a non-status response is returned, set the self link
if _, ok := result.(*api.Status); !ok {
if err := setSelfLink(result, req, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
}
}
write(http.StatusOK, scope.APIVersion, scope.Codec, result, w, req.Request)
}
}
// UpdateResource returns a function that will handle a resource update
func UpdateResource(r rest.Updater, scope RequestScope, typer runtime.ObjectTyper, admit admission.Interface) restful.RouteFunction {
return func(req *restful.Request, res *restful.Response) {
w := res.ResponseWriter
// TODO: we either want to remove timeout or document it (if we document, move timeout out of this function and declare it in api_installer)
timeout := parseTimeout(req.Request.URL.Query().Get("timeout"))
namespace, name, err := scope.Namer.Name(req)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
ctx := scope.ContextFunc(req)
ctx = api.WithNamespace(ctx, namespace)
body, err := readBody(req.Request)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
obj := r.New()
if err := scope.Codec.DecodeInto(body, obj); err != nil {
err = transformDecodeError(typer, err, obj, body)
errorJSON(err, scope.Codec, w)
return
}
if err := checkName(obj, name, namespace, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
if admit.Handles(admission.Update) {
userInfo, _ := api.UserFrom(ctx)
err = admit.Admit(admission.NewAttributesRecord(obj, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Update, userInfo))
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
}
wasCreated := false
result, err := finishRequest(timeout, func() (runtime.Object, error) {
obj, created, err := r.Update(ctx, obj)
wasCreated = created
return obj, err
})
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if err := setSelfLink(result, req, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
status := http.StatusOK
if wasCreated {
status = http.StatusCreated
}
writeJSON(status, scope.Codec, result, w, isPrettyPrint(req.Request))
}
}
// PatchResource returns a function that will handle a resource patch
// TODO: Eventually PatchResource should just use GuaranteedUpdate and this routine should be a bit cleaner
func PatchResource(r rest.Patcher, scope RequestScope, typer runtime.ObjectTyper, admit admission.Interface, converter runtime.ObjectConvertor) restful.RouteFunction {
return func(req *restful.Request, res *restful.Response) {
w := res.ResponseWriter
// TODO: we either want to remove timeout or document it (if we
// document, move timeout out of this function and declare it in
// api_installer)
timeout := parseTimeout(req.Request.URL.Query().Get("timeout"))
namespace, name, err := scope.Namer.Name(req)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
obj := r.New()
ctx := scope.ContextFunc(req)
ctx = api.WithNamespace(ctx, namespace)
// PATCH requires same permission as UPDATE
if admit.Handles(admission.Update) {
userInfo, _ := api.UserFrom(ctx)
err = admit.Admit(admission.NewAttributesRecord(obj, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Update, userInfo))
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
}
versionedObj, err := converter.ConvertToVersion(obj, scope.APIVersion)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
original, err := r.Get(ctx, name)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
originalObjJS, err := scope.Codec.Encode(original)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
patchJS, err := readBody(req.Request)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
contentType := req.HeaderParameter("Content-Type")
patchedObjJS, err := getPatchedJS(contentType, originalObjJS, patchJS, versionedObj)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if err := scope.Codec.DecodeInto(patchedObjJS, obj); err != nil {
errorJSON(err, scope.Codec, w)
return
}
if err := checkName(obj, name, namespace, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
result, err := finishRequest(timeout, func() (runtime.Object, error) {
// update should never create as previous get would fail
obj, _, err := r.Update(ctx, obj)
return obj, err
})
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if err := setSelfLink(result, req, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
write(http.StatusOK, scope.APIVersion, scope.Codec, result, w, req.Request)
}
}
func createHandler(r rest.NamedCreater, scope RequestScope, typer runtime.ObjectTyper, admit admission.Interface, includeName bool) restful.RouteFunction {
return func(req *restful.Request, res *restful.Response) {
w := res.ResponseWriter
// TODO: we either want to remove timeout or document it (if we document, move timeout out of this function and declare it in api_installer)
timeout := parseTimeout(req.Request.URL.Query().Get("timeout"))
var (
namespace, name string
err error
)
if includeName {
namespace, name, err = scope.Namer.Name(req)
} else {
namespace, err = scope.Namer.Namespace(req)
}
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
ctx := scope.ContextFunc(req)
ctx = api.WithNamespace(ctx, namespace)
body, err := readBody(req.Request)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
obj := r.New()
if err := scope.Codec.DecodeInto(body, obj); err != nil {
err = transformDecodeError(typer, err, obj, body)
errorJSON(err, scope.Codec, w)
return
}
if admit.Handles(admission.Create) {
userInfo, _ := api.UserFrom(ctx)
err = admit.Admit(admission.NewAttributesRecord(obj, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, userInfo))
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
}
result, err := finishRequest(timeout, func() (runtime.Object, error) {
out, err := r.Create(ctx, name, obj)
if status, ok := out.(*api.Status); ok && err == nil && status.Code == 0 {
status.Code = http.StatusCreated
}
return out, err
})
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if err := setSelfLink(result, req, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
write(http.StatusCreated, scope.APIVersion, scope.Codec, result, w, req.Request)
}
}
// ListResource returns a function that handles retrieving a list of resources from a rest.Storage object.
func ListResource(r rest.Lister, rw rest.Watcher, scope RequestScope, forceWatch bool, minRequestTimeout time.Duration) restful.RouteFunction {
return func(req *restful.Request, res *restful.Response) {
w := res.ResponseWriter
namespace, err := scope.Namer.Namespace(req)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
// Watches for single objects are routed to this function.
// Treat a /name parameter the same as a field selector entry.
hasName := true
_, name, err := scope.Namer.Name(req)
if err != nil {
hasName = false
}
ctx := scope.ContextFunc(req)
ctx = api.WithNamespace(ctx, namespace)
out, err := queryToObject(req.Request.URL.Query(), scope, "ListOptions")
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
opts := *out.(*api.ListOptions)
// transform fields
// TODO: queryToObject should do this.
fn := func(label, value string) (newLabel, newValue string, err error) {
return scope.Convertor.ConvertFieldLabel(scope.APIVersion, scope.Kind, label, value)
}
if opts.FieldSelector, err = opts.FieldSelector.Transform(fn); err != nil {
// TODO: allow bad request to set field causes based on query parameters
err = errors.NewBadRequest(err.Error())
errorJSON(err, scope.Codec, w)
return
}
if hasName {
// metadata.name is the canonical internal name.
// generic.SelectionPredicate will notice that this is
// a request for a single object and optimize the
// storage query accordingly.
nameSelector := fields.OneTermEqualSelector("metadata.name", name)
if opts.FieldSelector != nil && !opts.FieldSelector.Empty() {
// It doesn't make sense to ask for both a name
// and a field selector, since just the name is
// sufficient to narrow down the request to a
// single object.
errorJSON(
errors.NewBadRequest("both a name and a field selector provided; please provide one or the other."),
scope.Codec,
w,
)
return
}
opts.FieldSelector = nameSelector
}
if (opts.Watch || forceWatch) && rw != nil {
watcher, err := rw.Watch(ctx, opts.LabelSelector, opts.FieldSelector, opts.ResourceVersion)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
serveWatch(watcher, scope, w, req, minRequestTimeout)
return
}
result, err := r.List(ctx, opts.LabelSelector, opts.FieldSelector)
if err != nil {
errorJSON(err, scope.Codec, w)
return
}
if err := setListSelfLink(result, req, scope.Namer); err != nil {
errorJSON(err, scope.Codec, w)
return
}
write(http.StatusOK, scope.APIVersion, scope.Codec, result, w, req.Request)
}
}
func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var verb string
var apiResource string
var httpCode int
reqStart := time.Now()
defer metrics.Monitor(&verb, &apiResource, util.GetClient(req), &httpCode, reqStart)
requestInfo, err := r.apiRequestInfoResolver.GetAPIRequestInfo(req)
if err != nil {
notFound(w, req)
httpCode = http.StatusNotFound
return
}
verb = requestInfo.Verb
namespace, resource, parts := requestInfo.Namespace, requestInfo.Resource, requestInfo.Parts
ctx, ok := r.context.Get(req)
if !ok {
ctx = api.NewContext()
}
ctx = api.WithNamespace(ctx, namespace)
if len(parts) < 2 {
notFound(w, req)
httpCode = http.StatusNotFound
return
}
id := parts[1]
remainder := ""
if len(parts) > 2 {
proxyParts := parts[2:]
remainder = strings.Join(proxyParts, "/")
if strings.HasSuffix(req.URL.Path, "/") {
// The original path had a trailing slash, which has been stripped
// by KindAndNamespace(). We should add it back because some
// servers (like etcd) require it.
remainder = remainder + "/"
}
}
storage, ok := r.storage[resource]
if !ok {
httplog.LogOf(req, w).Addf("'%v' has no storage object", resource)
notFound(w, req)
httpCode = http.StatusNotFound
return
}
apiResource = resource
redirector, ok := storage.(rest.Redirector)
if !ok {
httplog.LogOf(req, w).Addf("'%v' is not a redirector", resource)
httpCode = errorJSON(errors.NewMethodNotSupported(resource, "proxy"), r.codec, w)
return
}
location, transport, err := redirector.ResourceLocation(ctx, id)
if err != nil {
httplog.LogOf(req, w).Addf("Error getting ResourceLocation: %v", err)
status := errToAPIStatus(err)
writeJSON(status.Code, r.codec, status, w, true)
httpCode = status.Code
return
}
if location == nil {
httplog.LogOf(req, w).Addf("ResourceLocation for %v returned nil", id)
notFound(w, req)
httpCode = http.StatusNotFound
return
}
// If we have a custom dialer, and no pre-existing transport, initialize it to use the dialer.
if transport == nil && r.dial != nil {
transport = &http.Transport{Dial: r.dial}
}
// Default to http
if location.Scheme == "" {
location.Scheme = "http"
}
// Add the subpath
if len(remainder) > 0 {
location.Path = singleJoiningSlash(location.Path, remainder)
}
// Start with anything returned from the storage, and add the original request's parameters
values := location.Query()
for k, vs := range req.URL.Query() {
for _, v := range vs {
values.Add(k, v)
}
}
location.RawQuery = values.Encode()
newReq, err := http.NewRequest(req.Method, location.String(), req.Body)
if err != nil {
status := errToAPIStatus(err)
writeJSON(status.Code, r.codec, status, w, true)
notFound(w, req)
httpCode = status.Code
return
}
httpCode = http.StatusOK
newReq.Header = req.Header
// TODO convert this entire proxy to an UpgradeAwareProxy similar to
// https://github.com/openshift/origin/blob/master/pkg/util/httpproxy/upgradeawareproxy.go.
// That proxy needs to be modified to support multiple backends, not just 1.
if r.tryUpgrade(w, req, newReq, location, transport) {
return
}
// Redirect requests of the form "/{resource}/{name}" to "/{resource}/{name}/"
// This is essentially a hack for https://github.com/qingyuancloud/QingYuan/issues/4958.
// Note: Keep this code after tryUpgrade to not break that flow.
if len(parts) == 2 && !strings.HasSuffix(req.URL.Path, "/") {
var queryPart string
if len(req.URL.RawQuery) > 0 {
queryPart = "?" + req.URL.RawQuery
}
w.Header().Set("Location", req.URL.Path+"/"+queryPart)
w.WriteHeader(http.StatusMovedPermanently)
return
}
proxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: location.Scheme, Host: location.Host})
if transport == nil {
prepend := path.Join(r.prefix, resource, id)
if len(namespace) > 0 {
prepend = path.Join(r.prefix, "namespaces", namespace, resource, id)
}
transport = &proxyutil.Transport{
Scheme: req.URL.Scheme,
Host: req.URL.Host,
PathPrepend: prepend,
}
}
proxy.Transport = transport
proxy.FlushInterval = 200 * time.Millisecond
proxy.ServeHTTP(w, newReq)
}