func TestGetMultipleTypeObjectsAsList(t *testing.T) {
pods, svc, _ := testData()
f, tf, codec, _ := cmdtesting.NewAPIFactory()
tf.Printer = &testPrinter{}
tf.Client = &fake.RESTClient{
NegotiatedSerializer: unstructuredSerializer,
Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
switch req.URL.Path {
case "/namespaces/test/pods":
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, pods)}, nil
case "/namespaces/test/services":
return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, svc)}, nil
default:
t.Fatalf("unexpected request: %#v\n%#v", req.URL, req)
return nil, nil
}
}),
}
tf.Namespace = "test"
tf.ClientConfig = &restclient.Config{ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(api.GroupName).GroupVersion}}
buf := bytes.NewBuffer([]byte{})
errBuf := bytes.NewBuffer([]byte{})
cmd := NewCmdGet(f, buf, errBuf)
cmd.SetOutput(buf)
cmd.Flags().Set("output", "json")
cmd.Run(cmd, []string{"pods,services"})
if tf.Printer.(*testPrinter).Objects != nil {
t.Errorf("unexpected print to default printer")
}
out, err := runtime.Decode(codec, buf.Bytes())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
list, err := meta.ExtractList(out)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if errs := runtime.DecodeList(list, codec); len(errs) > 0 {
t.Fatalf("unexpected error: %v", errs)
}
if err := meta.SetList(out, list); err != nil {
t.Fatalf("unexpected error: %v", err)
}
expected := &api.List{
Items: []runtime.Object{
&pods.Items[0],
&pods.Items[1],
&svc.Items[0],
},
}
if !reflect.DeepEqual(expected, out) {
t.Errorf("unexpected output: %#v", out)
}
}
func (s *SortingPrinter) sortObj(obj runtime.Object) error {
objs, err := meta.ExtractList(obj)
if err != nil {
return err
}
if len(objs) == 0 {
return nil
}
sorter, err := SortObjects(s.Decoder, objs, s.SortField)
if err != nil {
return err
}
switch list := obj.(type) {
case *v1.List:
outputList := make([]runtime.RawExtension, len(objs))
for ix := range objs {
outputList[ix] = list.Items[sorter.OriginalPosition(ix)]
}
list.Items = outputList
return nil
}
return meta.SetList(obj, objs)
}
// UsageStats calculates latest observed usage stats for all objects
func (g *GenericEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
// default each tracked resource to zero
result := quota.UsageStats{Used: api.ResourceList{}}
for _, resourceName := range g.MatchedResourceNames {
result.Used[resourceName] = resource.MustParse("0")
}
list, err := g.ListFuncByNamespace(options.Namespace, api.ListOptions{})
if err != nil {
return result, fmt.Errorf("%s: Failed to list %v: %v", g.Name, g.GroupKind(), err)
}
_, err = meta.Accessor(list)
if err != nil {
return result, fmt.Errorf("%s: Unable to understand list result %#v", g.Name, list)
}
items, err := meta.ExtractList(list)
if err != nil {
return result, fmt.Errorf("%s: Unable to understand list result %#v (%v)", g.Name, list, err)
}
for _, item := range items {
// need to verify that the item matches the set of scopes
matchesScopes := true
for _, scope := range options.Scopes {
if !g.MatchesScope(scope, item) {
matchesScopes = false
}
}
// only count usage if there was a match
if matchesScopes {
result.Used = quota.Add(result.Used, g.Usage(item))
}
}
return result, nil
}
func (v FlattenListVisitor) Visit(fn VisitorFunc) error {
return v.Visitor.Visit(func(info *Info, err error) error {
if err != nil {
return err
}
if info.Object == nil {
return fn(info, nil)
}
items, err := meta.ExtractList(info.Object)
if err != nil {
return fn(info, nil)
}
if errs := runtime.DecodeList(items, struct {
runtime.ObjectTyper
runtime.Decoder
}{v.Mapper, info.Mapping.Codec}); len(errs) > 0 {
return utilerrors.NewAggregate(errs)
}
for i := range items {
item, err := v.InfoForObject(items[i])
if err != nil {
return err
}
if len(info.ResourceVersion) != 0 {
item.ResourceVersion = info.ResourceVersion
}
if err := fn(item, nil); err != nil {
return err
}
}
return nil
})
}
func (o objects) Add(obj runtime.Object) error {
gvk, err := o.scheme.ObjectKind(obj)
if err != nil {
return err
}
kind := gvk.Kind
switch {
case meta.IsListType(obj):
if kind != "List" {
o.types[kind] = append(o.types[kind], obj)
}
list, err := meta.ExtractList(obj)
if err != nil {
return err
}
if errs := runtime.DecodeList(list, o.decoder); len(errs) > 0 {
return errs[0]
}
for _, obj := range list {
if err := o.Add(obj); err != nil {
return err
}
}
default:
if status, ok := obj.(*unversioned.Status); ok && status.Details != nil {
kind = status.Details.Kind
}
o.types[kind] = append(o.types[kind], obj)
}
return nil
}
// ReadObjectsFromPath reads objects from the specified file for testing.
func ReadObjectsFromPath(path, namespace string, decoder runtime.Decoder, typer runtime.ObjectTyper) ([]runtime.Object, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
data, err = yaml.ToJSON(data)
if err != nil {
return nil, err
}
obj, err := runtime.Decode(decoder, data)
if err != nil {
return nil, err
}
if !meta.IsListType(obj) {
if err := setNamespace(typer, obj, namespace); err != nil {
return nil, err
}
return []runtime.Object{obj}, nil
}
list, err := meta.ExtractList(obj)
if err != nil {
return nil, err
}
errs := runtime.DecodeList(list, decoder)
if len(errs) > 0 {
return nil, errs[0]
}
for _, o := range list {
if err := setNamespace(typer, o, namespace); err != nil {
return nil, err
}
}
return list, nil
}
func (s *CustomColumnsPrinter) PrintObj(obj runtime.Object, out io.Writer) error {
w := tabwriter.NewWriter(out, columnwidth, tabwidth, padding, padding_character, flags)
headers := make([]string, len(s.Columns))
for ix := range s.Columns {
headers[ix] = s.Columns[ix].Header
}
fmt.Fprintln(w, strings.Join(headers, "\t"))
parsers := make([]*jsonpath.JSONPath, len(s.Columns))
for ix := range s.Columns {
parsers[ix] = jsonpath.New(fmt.Sprintf("column%d", ix))
if err := parsers[ix].Parse(s.Columns[ix].FieldSpec); err != nil {
return err
}
}
if meta.IsListType(obj) {
objs, err := meta.ExtractList(obj)
if err != nil {
return err
}
for ix := range objs {
if err := s.printOneObject(objs[ix], parsers, w); err != nil {
return err
}
}
} else {
if err := s.printOneObject(obj, parsers, w); err != nil {
return err
}
}
return w.Flush()
}
// filterList filters any list object that conforms to the api conventions,
// provided that 'm' works with the concrete type of list. d is an optional
// decorator for the returned functions. Only matching items are decorated.
func filterList(list runtime.Object, m generic.Matcher, d decoratorFunc) (filtered runtime.Object, err error) {
// TODO: push a matcher down into tools.etcdHelper to avoid all this
// nonsense. This is a lot of unnecessary copies.
items, err := meta.ExtractList(list)
if err != nil {
return nil, err
}
var filteredItems []runtime.Object
for _, obj := range items {
match, err := m.Matches(obj)
if err != nil {
return nil, err
}
if match {
if d != nil {
if err := d(obj); err != nil {
return nil, err
}
}
filteredItems = append(filteredItems, obj)
}
}
err = meta.SetList(list, filteredItems)
if err != nil {
return nil, err
}
return list, nil
}
// setListSelfLink sets the self link of a list to the base URL, then sets the self links
// on all child objects returned.
func setListSelfLink(obj runtime.Object, req *restful.Request, namer ScopeNamer) error {
if !meta.IsListType(obj) {
return nil
}
// TODO: List SelfLink generation should return a full URL?
path, query, err := namer.GenerateListLink(req)
if err != nil {
return err
}
newURL := *req.Request.URL
newURL.Path = path
newURL.RawQuery = query
// use the path that got us here
newURL.Fragment = ""
if err := namer.SetSelfLink(obj, newURL.String()); err != nil {
glog.V(4).Infof("Unable to set self link on object: %v", err)
}
// Set self-link of objects in the list.
items, err := meta.ExtractList(obj)
if err != nil {
return err
}
for i := range items {
if err := setSelfLink(items[i], req, namer); err != nil {
return err
}
}
return meta.SetList(obj, items)
}
func TestExtractListGenericV1(t *testing.T) {
pl := &v1.List{
Items: []runtime.RawExtension{
{Raw: []byte("foo")},
{Raw: []byte("bar")},
{Object: &v1.Pod{ObjectMeta: v1.ObjectMeta{Name: "other"}}},
},
}
list, err := meta.ExtractList(pl)
if err != nil {
t.Fatalf("Unexpected error %v", err)
}
if e, a := len(list), len(pl.Items); e != a {
t.Fatalf("Expected %v, got %v", e, a)
}
if obj, ok := list[0].(*runtime.Unknown); !ok {
t.Fatalf("Expected list[0] to be *runtime.Unknown, it is %#v", obj)
}
if obj, ok := list[1].(*runtime.Unknown); !ok {
t.Fatalf("Expected list[1] to be *runtime.Unknown, it is %#v", obj)
}
if obj, ok := list[2].(*v1.Pod); !ok {
t.Fatalf("Expected list[2] to be *runtime.Unknown, it is %#v", obj)
}
}
// CreateList will properly create a list using the storage interface
func CreateList(prefix string, helper storage.Interface, list runtime.Object) error {
items, err := meta.ExtractList(list)
if err != nil {
return err
}
err = CreateObjList(prefix, helper, items)
if err != nil {
return err
}
return meta.SetList(list, items)
}
func TestExtractListOfInterfacePtrs(t *testing.T) {
pl := &fakePtrInterfaceList{
Items: &[]runtime.Object{},
}
list, err := meta.ExtractList(pl)
if err != nil {
t.Fatalf("Unexpected error %v", err)
}
if len(list) > 0 {
t.Fatalf("Expected empty list, got %#v", list)
}
}
func extractResourceList(objs []runtime.Object) ([]runtime.Object, error) {
finalObjs := []runtime.Object{}
for _, obj := range objs {
items, err := meta.ExtractList(obj)
if err != nil {
return nil, err
}
for _, item := range items {
finalObjs = append(finalObjs, item)
}
}
return finalObjs, nil
}
func verifyListMetadata(t *testing.T, metaOnlyList *MetadataOnlyObjectList) {
items, err := meta.ExtractList(metaOnlyList)
if err != nil {
t.Fatal(err)
}
for _, item := range items {
metaOnly, ok := item.(*MetadataOnlyObject)
if !ok {
t.Fatalf("expected item to be *MetadataOnlyObject")
}
verfiyMetadata("check list", t, metaOnly)
}
}
func validateObject(path string, obj runtime.Object, t *testing.T) {
// if an object requires a namespace server side, be sure that it is filled in for validation
if validation.HasObjectMeta(obj) {
namespaceRequired, err := validation.GetRequiresNamespace(obj)
if err != nil {
t.Errorf("Expected no error, Got %v", err)
return
}
if namespaceRequired {
objectMeta, objectMetaErr := kapi.ObjectMetaFor(obj)
if objectMetaErr != nil {
t.Errorf("Expected no error, Got %v", objectMetaErr)
return
}
objectMeta.Namespace = kapi.NamespaceDefault
}
}
switch typedObj := obj.(type) {
case *kapi.Pod:
if errors := kvalidation.ValidatePod(typedObj); len(errors) > 0 {
t.Errorf("%s did not validate correctly: %v", path, errors)
}
case *kapi.Service:
if errors := kvalidation.ValidateService(typedObj); len(errors) > 0 {
t.Errorf("%s did not validate correctly: %v", path, errors)
}
case *kapi.List, *imageapi.ImageStreamList:
if list, err := meta.ExtractList(typedObj); err == nil {
runtime.DecodeList(list, kapi.Codecs.UniversalDecoder())
for i := range list {
validateObject(path, list[i], t)
}
} else {
t.Errorf("Expected no error, Got %v", err)
}
default:
if errors := validation.Validator.Validate(obj); len(errors) > 0 {
t.Errorf("%s with %v did not validate correctly: %v", path, reflect.TypeOf(obj), errors)
}
}
}
func (t *tracker) addList(obj runtime.Object, replaceExisting bool) error {
list, err := meta.ExtractList(obj)
if err != nil {
return err
}
errs := runtime.DecodeList(list, t.decoder)
if len(errs) > 0 {
return errs[0]
}
for _, obj := range list {
err := t.add(obj, replaceExisting)
if err != nil {
return err
}
}
return nil
}
func (p *pruner) prune(namespace string, mapping *meta.RESTMapping, shortOutput bool) error {
c, err := p.clientFunc(mapping)
if err != nil {
return err
}
objList, err := resource.NewHelper(c, mapping).List(namespace, mapping.GroupVersionKind.Version, p.selector, false)
if err != nil {
return err
}
objs, err := meta.ExtractList(objList)
if err != nil {
return err
}
for _, obj := range objs {
annots, err := mapping.MetadataAccessor.Annotations(obj)
if err != nil {
return err
}
if _, ok := annots[annotations.LastAppliedConfigAnnotation]; !ok {
// don't prune resources not created with apply
continue
}
uid, err := mapping.UID(obj)
if err != nil {
return err
}
if p.visitedUids.Has(string(uid)) {
continue
}
name, err := mapping.Name(obj)
if err != nil {
return err
}
if !p.dryRun {
if err := p.delete(namespace, name, mapping, c); err != nil {
return err
}
}
cmdutil.PrintSuccess(p.mapper, shortOutput, p.out, mapping.Resource, name, p.dryRun, "pruned")
}
return nil
}
func TestExtractListOfValuePtrs(t *testing.T) {
pl := &fakePtrValueList{
Items: []*api.Pod{
{ObjectMeta: api.ObjectMeta{Name: "1"}},
{ObjectMeta: api.ObjectMeta{Name: "2"}},
},
}
list, err := meta.ExtractList(pl)
if err != nil {
t.Fatalf("Unexpected error %v", err)
}
if e, a := len(list), len(pl.Items); e != a {
t.Fatalf("Expected %v, got %v", e, a)
}
for i := range list {
if obj, ok := list[i].(*api.Pod); !ok {
t.Fatalf("Expected list[%d] to be *api.Pod, it is %#v", i, obj)
}
}
}
// DeleteCollection remove all items returned by List with a given ListOptions from etcd.
//
// DeleteCollection is currently NOT atomic. It can happen that only subset of objects
// will be deleted from etcd, and then an error will be returned.
// In case of success, the list of deleted objects will be returned.
//
// TODO: Currently, there is no easy way to remove 'directory' entry from etcd (if we
// are removing all objects of a given type) with the current API (it's technically
// possibly with etcd API, but watch is not delivered correctly then).
// It will be possible to fix it with v3 etcd API.
func (e *Etcd) DeleteCollection(ctx api.Context, options *api.DeleteOptions, listOptions *unversioned.ListOptions) (runtime.Object, error) {
listObj, err := e.List(ctx, listOptions)
if err != nil {
return nil, err
}
items, err := meta.ExtractList(listObj)
if err != nil {
return nil, err
}
for _, item := range items {
accessor, err := meta.Accessor(item)
if err != nil {
return nil, err
}
if _, err := e.Delete(ctx, accessor.GetName(), options); err != nil {
return nil, err
}
}
return listObj, nil
}
func TestExtractListGeneric(t *testing.T) {
pl := &api.List{
Items: []runtime.Object{
&api.Pod{ObjectMeta: api.ObjectMeta{Name: "1"}},
&api.Service{ObjectMeta: api.ObjectMeta{Name: "2"}},
},
}
list, err := meta.ExtractList(pl)
if err != nil {
t.Fatalf("Unexpected error %v", err)
}
if e, a := len(list), len(pl.Items); e != a {
t.Fatalf("Expected %v, got %v", e, a)
}
if obj, ok := list[0].(*api.Pod); !ok {
t.Fatalf("Expected list[0] to be *api.Pod, it is %#v", obj)
}
if obj, ok := list[1].(*api.Service); !ok {
t.Fatalf("Expected list[1] to be *api.Service, it is %#v", obj)
}
}
func TestExtractList(t *testing.T) {
pl := &api.PodList{
Items: []api.Pod{
{ObjectMeta: api.ObjectMeta{Name: "1"}},
{ObjectMeta: api.ObjectMeta{Name: "2"}},
{ObjectMeta: api.ObjectMeta{Name: "3"}},
},
}
list, err := meta.ExtractList(pl)
if err != nil {
t.Fatalf("Unexpected error %v", err)
}
if e, a := len(list), len(pl.Items); e != a {
t.Fatalf("Expected %v, got %v", e, a)
}
for i := range list {
if e, a := list[i].(*api.Pod).Name, pl.Items[i].Name; e != a {
t.Fatalf("Expected %v, got %v", e, a)
}
}
}
func (v FlattenListVisitor) Visit(fn VisitorFunc) error {
return v.Visitor.Visit(func(info *Info, err error) error {
if err != nil {
return err
}
if info.Object == nil {
return fn(info, nil)
}
items, err := meta.ExtractList(info.Object)
if err != nil {
return fn(info, nil)
}
if errs := runtime.DecodeList(items, struct {
runtime.ObjectTyper
runtime.Decoder
}{v.Mapper, v.Mapper.Decoder}); len(errs) > 0 {
return utilerrors.NewAggregate(errs)
}
// If we have a GroupVersionKind on the list, prioritize that when asking for info on the objects contained in the list
var preferredGVKs []unversioned.GroupVersionKind
if info.Mapping != nil && !info.Mapping.GroupVersionKind.Empty() {
preferredGVKs = append(preferredGVKs, info.Mapping.GroupVersionKind)
}
for i := range items {
item, err := v.InfoForObject(items[i], preferredGVKs)
if err != nil {
return err
}
if len(info.ResourceVersion) != 0 {
item.ResourceVersion = info.ResourceVersion
}
if err := fn(item, nil); err != nil {
return err
}
}
return nil
})
}
func TestSetExtractListRoundTrip(t *testing.T) {
fuzzer := fuzz.New().NilChance(0).NumElements(1, 5)
for i := 0; i < 5; i++ {
start := &api.PodList{}
fuzzer.Fuzz(&start.Items)
list, err := meta.ExtractList(start)
if err != nil {
t.Errorf("Unexpected error %v", err)
continue
}
got := &api.PodList{}
err = meta.SetList(got, list)
if err != nil {
t.Errorf("Unexpected error %v", err)
continue
}
if e, a := start, got; !reflect.DeepEqual(e, a) {
t.Fatalf("Expected %#v, got %#v", e, a)
}
}
}
// FilterResourceList receives a list of runtime objects.
// If any objects are filtered, that number is returned along with a modified list.
func FilterResourceList(obj runtime.Object, filterFuncs kubectl.Filters, filterOpts *kubectl.PrintOptions) (int, []runtime.Object, error) {
items, err := meta.ExtractList(obj)
if err != nil {
return 0, []runtime.Object{obj}, utilerrors.NewAggregate([]error{err})
}
if errs := runtime.DecodeList(items, api.Codecs.UniversalDecoder(), runtime.UnstructuredJSONScheme); len(errs) > 0 {
return 0, []runtime.Object{obj}, utilerrors.NewAggregate(errs)
}
filterCount := 0
list := make([]runtime.Object, 0, len(items))
for _, obj := range items {
if isFiltered, err := filterFuncs.Filter(obj, filterOpts); !isFiltered {
if err != nil {
glog.V(2).Infof("Unable to filter resource: %v", err)
continue
}
list = append(list, obj)
} else if isFiltered {
filterCount++
}
}
return filterCount, list, nil
}
// DeleteCollection remove all items returned by List with a given ListOptions from storage.
//
// DeleteCollection is currently NOT atomic. It can happen that only subset of objects
// will be deleted from storage, and then an error will be returned.
// In case of success, the list of deleted objects will be returned.
//
// TODO: Currently, there is no easy way to remove 'directory' entry from storage (if we
// are removing all objects of a given type) with the current API (it's technically
// possibly with storage API, but watch is not delivered correctly then).
// It will be possible to fix it with v3 etcd API.
func (e *Store) DeleteCollection(ctx api.Context, options *api.DeleteOptions, listOptions *api.ListOptions) (runtime.Object, error) {
listObj, err := e.List(ctx, listOptions)
if err != nil {
return nil, err
}
items, err := meta.ExtractList(listObj)
if err != nil {
return nil, err
}
// Spawn a number of goroutines, so that we can issue requests to storage
// in parallel to speed up deletion.
// TODO: Make this proportional to the number of items to delete, up to
// DeleteCollectionWorkers (it doesn't make much sense to spawn 16
// workers to delete 10 items).
workersNumber := e.DeleteCollectionWorkers
if workersNumber < 1 {
workersNumber = 1
}
wg := sync.WaitGroup{}
toProcess := make(chan int, 2*workersNumber)
errs := make(chan error, workersNumber+1)
go func() {
defer utilruntime.HandleCrash(func(panicReason interface{}) {
errs <- fmt.Errorf("DeleteCollection distributor panicked: %v", panicReason)
})
for i := 0; i < len(items); i++ {
toProcess <- i
}
close(toProcess)
}()
wg.Add(workersNumber)
for i := 0; i < workersNumber; i++ {
go func() {
// panics don't cross goroutine boundaries
defer utilruntime.HandleCrash(func(panicReason interface{}) {
errs <- fmt.Errorf("DeleteCollection goroutine panicked: %v", panicReason)
})
defer wg.Done()
for {
index, ok := <-toProcess
if !ok {
return
}
accessor, err := meta.Accessor(items[index])
if err != nil {
errs <- err
return
}
if _, err := e.Delete(ctx, accessor.GetName(), options); err != nil && !kubeerr.IsNotFound(err) {
glog.V(4).Infof("Delete %s in DeleteCollection failed: %v", accessor.GetName(), err)
errs <- err
return
}
}
}()
}
wg.Wait()
select {
case err := <-errs:
return nil, err
default:
return listObj, nil
}
}
func TestArrayOfRuntimeObject(t *testing.T) {
internalGV := unversioned.GroupVersion{Group: "test.group", Version: runtime.APIVersionInternal}
externalGV := unversioned.GroupVersion{Group: "test.group", Version: "v1test"}
s := runtime.NewScheme()
s.AddKnownTypes(internalGV, &EmbeddedTest{})
s.AddKnownTypeWithName(externalGV.WithKind("EmbeddedTest"), &EmbeddedTestExternal{})
s.AddKnownTypes(internalGV, &ObjectTest{})
s.AddKnownTypeWithName(externalGV.WithKind("ObjectTest"), &ObjectTestExternal{})
codec := serializer.NewCodecFactory(s).LegacyCodec(externalGV)
innerItems := []runtime.Object{
&EmbeddedTest{ID: "baz"},
}
items := []runtime.Object{
&EmbeddedTest{ID: "foo"},
&EmbeddedTest{ID: "bar"},
// TODO: until YAML is removed, this JSON must be in ascending key order to ensure consistent roundtrip serialization
&runtime.Unknown{
Raw: []byte(`{"apiVersion":"unknown.group/unknown","foo":"bar","kind":"OtherTest"}`),
ContentType: runtime.ContentTypeJSON,
},
&ObjectTest{
Items: runtime.NewEncodableList(codec, innerItems),
},
}
internal := &ObjectTest{
Items: runtime.NewEncodableList(codec, items),
}
wire, err := runtime.Encode(codec, internal)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Logf("Wire format is:\n%s\n", string(wire))
obj := &ObjectTestExternal{}
if err := json.Unmarshal(wire, obj); err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Logf("exact wire is: %s", string(obj.Items[0].Raw))
items[3] = &ObjectTest{Items: innerItems}
internal.Items = items
decoded, err := runtime.Decode(codec, wire)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
list, err := meta.ExtractList(decoded)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if errs := runtime.DecodeList(list, codec); len(errs) > 0 {
t.Fatalf("unexpected error: %v", errs)
}
list2, err := meta.ExtractList(list[3])
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if errs := runtime.DecodeList(list2, codec); len(errs) > 0 {
t.Fatalf("unexpected error: %v", errs)
}
if err := meta.SetList(list[3], list2); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// we want DecodeList to set type meta if possible, even on runtime.Unknown objects
internal.Items[2].(*runtime.Unknown).TypeMeta = runtime.TypeMeta{Kind: "OtherTest", APIVersion: "unknown.group/unknown"}
if e, a := internal.Items, list; !reflect.DeepEqual(e, a) {
t.Errorf("mismatched decoded: %s", diff.ObjectGoPrintSideBySide(e, a))
}
}
// ListAndWatch first lists all items and get the resource version at the moment of call,
// and then use the resource version to watch.
// It returns error if ListAndWatch didn't even try to initialize watch.
func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
glog.V(3).Infof("Listing and watching %v from %s", r.expectedType, r.name)
var resourceVersion string
resyncCh, cleanup := r.resyncChan()
defer cleanup()
// Explicitly set "0" as resource version - it's fine for the List()
// to be served from cache and potentially be delayed relative to
// etcd contents. Reflector framework will catch up via Watch() eventually.
options := api.ListOptions{ResourceVersion: "0"}
list, err := r.listerWatcher.List(options)
if err != nil {
return fmt.Errorf("%s: Failed to list %v: %v", r.name, r.expectedType, err)
}
listMetaInterface, err := meta.ListAccessor(list)
if err != nil {
return fmt.Errorf("%s: Unable to understand list result %#v: %v", r.name, list, err)
}
resourceVersion = listMetaInterface.GetResourceVersion()
items, err := meta.ExtractList(list)
if err != nil {
return fmt.Errorf("%s: Unable to understand list result %#v (%v)", r.name, list, err)
}
if err := r.syncWith(items, resourceVersion); err != nil {
return fmt.Errorf("%s: Unable to sync list result: %v", r.name, err)
}
r.setLastSyncResourceVersion(resourceVersion)
for {
options := api.ListOptions{
ResourceVersion: resourceVersion,
// We want to avoid situations when resyncing is breaking the TCP connection
// - see comment for 'timeoutForWatch()' for more details.
TimeoutSeconds: r.timeoutForWatch(),
}
w, err := r.listerWatcher.Watch(options)
if err != nil {
switch err {
case io.EOF:
// watch closed normally
case io.ErrUnexpectedEOF:
glog.V(1).Infof("%s: Watch for %v closed with unexpected EOF: %v", r.name, r.expectedType, err)
default:
utilruntime.HandleError(fmt.Errorf("%s: Failed to watch %v: %v", r.name, r.expectedType, err))
}
// If this is "connection refused" error, it means that most likely apiserver is not responsive.
// It doesn't make sense to re-list all objects because most likely we will be able to restart
// watch where we ended.
// If that's the case wait and resend watch request.
if urlError, ok := err.(*url.Error); ok {
if opError, ok := urlError.Err.(*net.OpError); ok {
if errno, ok := opError.Err.(syscall.Errno); ok && errno == syscall.ECONNREFUSED {
time.Sleep(time.Second)
continue
}
}
}
return nil
}
if err := r.watchHandler(w, &resourceVersion, resyncCh, stopCh); err != nil {
if err != errorResyncRequested && err != errorStopRequested {
glog.Warningf("%s: watch of %v ended with: %v", r.name, r.expectedType, err)
}
return nil
}
if r.canForceResyncNow() {
glog.V(4).Infof("%s: next resync planned for %#v, forcing now", r.name, r.nextResync)
return nil
}
}
}
func RunEdit(f *cmdutil.Factory, out, errOut io.Writer, cmd *cobra.Command, args []string, options *EditOptions) error {
var printer kubectl.ResourcePrinter
var ext string
switch format := cmdutil.GetFlagString(cmd, "output"); format {
case "json":
printer = &kubectl.JSONPrinter{}
ext = ".json"
case "yaml":
printer = &kubectl.YAMLPrinter{}
ext = ".yaml"
default:
return cmdutil.UsageError(cmd, "The flag 'output' must be one of yaml|json")
}
cmdNamespace, enforceNamespace, err := f.DefaultNamespace()
if err != nil {
return err
}
mapper, typer := f.Object(cmdutil.GetIncludeThirdPartyAPIs(cmd))
resourceMapper := &resource.Mapper{
ObjectTyper: typer,
RESTMapper: mapper,
ClientMapper: resource.ClientMapperFunc(f.ClientForMapping),
// NB: we use `f.Decoder(false)` to get a plain deserializer for
// the resourceMapper, since it's used to read in edits and
// we don't want to convert into the internal version when
// reading in edits (this would cause us to potentially try to
// compare two different GroupVersions).
Decoder: f.Decoder(false),
}
r := resource.NewBuilder(mapper, typer, resource.ClientMapperFunc(f.ClientForMapping), f.Decoder(true)).
NamespaceParam(cmdNamespace).DefaultNamespace().
FilenameParam(enforceNamespace, options.Recursive, options.Filenames...).
ResourceTypeOrNameArgs(true, args...).
Flatten().
Latest().
Do()
err = r.Err()
if err != nil {
return err
}
infos, err := r.Infos()
if err != nil {
return err
}
clientConfig, err := f.ClientConfig()
if err != nil {
return err
}
encoder := f.JSONEncoder()
defaultVersion, err := cmdutil.OutputVersion(cmd, clientConfig.GroupVersion)
if err != nil {
return err
}
originalObj, err := resource.AsVersionedObject(infos, false, defaultVersion, encoder)
if err != nil {
return err
}
var (
windowsLineEndings = cmdutil.GetFlagBool(cmd, "windows-line-endings")
edit = editor.NewDefaultEditor(f.EditorEnvs())
results = editResults{}
original = []byte{}
edited = []byte{}
file string
)
containsError := false
for {
// infos mutates over time to be the list of things we've tried and failed to edit
// this means that our overall list changes over time.
objToEdit, err := resource.AsVersionedObject(infos, false, defaultVersion, encoder)
if err != nil {
return err
}
// generate the file to edit
buf := &bytes.Buffer{}
var w io.Writer = buf
if windowsLineEndings {
w = crlf.NewCRLFWriter(w)
}
if err := results.header.writeTo(w); err != nil {
return preservedFile(err, results.file, errOut)
}
if !containsError {
if err := printer.PrintObj(objToEdit, w); err != nil {
return preservedFile(err, results.file, errOut)
}
original = buf.Bytes()
} else {
// In case of an error, preserve the edited file.
// Remove the comments (header) from it since we already
// have included the latest header in the buffer above.
buf.Write(manualStrip(edited))
}
// launch the editor
editedDiff := edited
edited, file, err = edit.LaunchTempFile(fmt.Sprintf("%s-edit-", filepath.Base(os.Args[0])), ext, buf)
if err != nil {
return preservedFile(err, results.file, errOut)
}
if bytes.Equal(stripComments(editedDiff), stripComments(edited)) {
// Ugly hack right here. We will hit this either (1) when we try to
// save the same changes we tried to save in the previous iteration
// which means our changes are invalid or (2) when we exit the second
// time. The second case is more usual so we can probably live with it.
// TODO: A less hacky fix would be welcome :)
fmt.Fprintln(errOut, "Edit cancelled, no valid changes were saved.")
return nil
}
// cleanup any file from the previous pass
if len(results.file) > 0 {
os.Remove(results.file)
}
glog.V(4).Infof("User edited:\n%s", string(edited))
// Compare content without comments
if bytes.Equal(stripComments(original), stripComments(edited)) {
os.Remove(file)
fmt.Fprintln(errOut, "Edit cancelled, no changes made.")
return nil
}
lines, err := hasLines(bytes.NewBuffer(edited))
if err != nil {
return preservedFile(err, file, errOut)
}
if !lines {
os.Remove(file)
fmt.Fprintln(errOut, "Edit cancelled, saved file was empty.")
return nil
}
results = editResults{
file: file,
}
// parse the edited file
updates, err := resourceMapper.InfoForData(edited, "edited-file")
if err != nil {
// syntax error
containsError = true
results.header.reasons = append(results.header.reasons, editReason{head: fmt.Sprintf("The edited file had a syntax error: %v", err)})
continue
}
// not a syntax error as it turns out...
containsError = false
namespaceVisitor := resource.NewFlattenListVisitor(updates, resourceMapper)
// need to make sure the original namespace wasn't changed while editing
if err = namespaceVisitor.Visit(resource.RequireNamespace(cmdNamespace)); err != nil {
return preservedFile(err, file, errOut)
}
mutatedObjects := []runtime.Object{}
annotationVisitor := resource.NewFlattenListVisitor(updates, resourceMapper)
// iterate through all items to apply annotations
if err = annotationVisitor.Visit(func(info *resource.Info, incomingErr error) error {
// put configuration annotation in "updates"
if err := kubectl.CreateOrUpdateAnnotation(cmdutil.GetFlagBool(cmd, cmdutil.ApplyAnnotationsFlag), info, encoder); err != nil {
return err
}
if cmdutil.ShouldRecord(cmd, info) {
if err := cmdutil.RecordChangeCause(info.Object, f.Command()); err != nil {
return err
}
}
mutatedObjects = append(mutatedObjects, info.Object)
return nil
}); err != nil {
return preservedFile(err, file, errOut)
}
// if we mutated a list in the visitor, persist the changes on the overall object
if meta.IsListType(updates.Object) {
meta.SetList(updates.Object, mutatedObjects)
}
patchVisitor := resource.NewFlattenListVisitor(updates, resourceMapper)
err = patchVisitor.Visit(func(info *resource.Info, incomingErr error) error {
currOriginalObj := originalObj
// if we're editing a list, then navigate the list to find the item that we're currently trying to edit
if meta.IsListType(originalObj) {
currOriginalObj = nil
editObjUID, err := meta.NewAccessor().UID(info.Object)
if err != nil {
return err
}
listItems, err := meta.ExtractList(originalObj)
if err != nil {
return err
}
// iterate through the list to find the item with the matching UID
for i := range listItems {
originalObjUID, err := meta.NewAccessor().UID(listItems[i])
if err != nil {
return err
}
if editObjUID == originalObjUID {
currOriginalObj = listItems[i]
break
}
}
if currOriginalObj == nil {
return fmt.Errorf("no original object found for %#v", info.Object)
}
}
originalSerialization, err := runtime.Encode(encoder, currOriginalObj)
if err != nil {
return err
}
editedSerialization, err := runtime.Encode(encoder, info.Object)
if err != nil {
return err
}
// compute the patch on a per-item basis
// use strategic merge to create a patch
originalJS, err := yaml.ToJSON(originalSerialization)
if err != nil {
return err
}
editedJS, err := yaml.ToJSON(editedSerialization)
if err != nil {
return err
}
if reflect.DeepEqual(originalJS, editedJS) {
// no edit, so just skip it.
cmdutil.PrintSuccess(mapper, false, out, info.Mapping.Resource, info.Name, "skipped")
return nil
}
patch, err := strategicpatch.CreateStrategicMergePatch(originalJS, editedJS, currOriginalObj)
// TODO: change all jsonmerge to strategicpatch
// for checking preconditions
preconditions := []jsonmerge.PreconditionFunc{}
if err != nil {
glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err)
return err
} else {
preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("apiVersion"))
preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("kind"))
preconditions = append(preconditions, jsonmerge.RequireMetadataKeyUnchanged("name"))
results.version = defaultVersion
}
if hold, msg := jsonmerge.TestPreconditionsHold(patch, preconditions); !hold {
fmt.Fprintf(errOut, "error: %s", msg)
return preservedFile(nil, file, errOut)
}
patched, err := resource.NewHelper(info.Client, info.Mapping).Patch(info.Namespace, info.Name, api.StrategicMergePatchType, patch)
if err != nil {
fmt.Fprintln(out, results.addError(err, info))
return nil
}
info.Refresh(patched, true)
cmdutil.PrintSuccess(mapper, false, out, info.Mapping.Resource, info.Name, "edited")
return nil
})
if err != nil {
return preservedFile(err, results.file, errOut)
}
// Handle all possible errors
//
// 1. retryable: propose kubectl replace -f
// 2. notfound: indicate the location of the saved configuration of the deleted resource
// 3. invalid: retry those on the spot by looping ie. reloading the editor
if results.retryable > 0 {
fmt.Fprintf(errOut, "You can run `%s replace -f %s` to try this update again.\n", filepath.Base(os.Args[0]), file)
return errExit
}
if results.notfound > 0 {
fmt.Fprintf(errOut, "The edits you made on deleted resources have been saved to %q\n", file)
return errExit
}
if len(results.edit) == 0 {
if results.notfound == 0 {
os.Remove(file)
} else {
fmt.Fprintf(out, "The edits you made on deleted resources have been saved to %q\n", file)
}
return nil
}
// loop again and edit the remaining items
infos = results.edit
}
}
// ListAndWatch first lists all items and get the resource version at the moment of call,
// and then use the resource version to watch.
// It returns error if ListAndWatch didn't even try to initialize watch.
func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
glog.V(3).Infof("Listing and watching %v from %s", r.expectedType, r.name)
var resourceVersion string
resyncCh, cleanup := r.resyncChan()
defer cleanup()
// Explicitly set "0" as resource version - it's fine for the List()
// to be served from cache and potentially be delayed relative to
// etcd contents. Reflector framework will catch up via Watch() eventually.
options := api.ListOptions{ResourceVersion: "0"}
list, err := r.listerWatcher.List(options)
if err != nil {
return fmt.Errorf("%s: Failed to list %v: %v", r.name, r.expectedType, err)
}
listMetaInterface, err := meta.ListAccessor(list)
if err != nil {
return fmt.Errorf("%s: Unable to understand list result %#v: %v", r.name, list, err)
}
resourceVersion = listMetaInterface.GetResourceVersion()
items, err := meta.ExtractList(list)
if err != nil {
return fmt.Errorf("%s: Unable to understand list result %#v (%v)", r.name, list, err)
}
if err := r.syncWith(items, resourceVersion); err != nil {
return fmt.Errorf("%s: Unable to sync list result: %v", r.name, err)
}
r.setLastSyncResourceVersion(resourceVersion)
resyncerrc := make(chan error, 1)
go func() {
for {
select {
case <-resyncCh:
case <-stopCh:
return
}
glog.V(4).Infof("%s: forcing resync", r.name)
if err := r.store.Resync(); err != nil {
resyncerrc <- err
return
}
cleanup()
resyncCh, cleanup = r.resyncChan()
}
}()
for {
timemoutseconds := int64(minWatchTimeout.Seconds() * (rand.Float64() + 1.0))
options = api.ListOptions{
ResourceVersion: resourceVersion,
// We want to avoid situations of hanging watchers. Stop any wachers that do not
// receive any events within the timeout window.
TimeoutSeconds: &timemoutseconds,
}
w, err := r.listerWatcher.Watch(options)
if err != nil {
switch err {
case io.EOF:
// watch closed normally
case io.ErrUnexpectedEOF:
glog.V(1).Infof("%s: Watch for %v closed with unexpected EOF: %v", r.name, r.expectedType, err)
default:
utilruntime.HandleError(fmt.Errorf("%s: Failed to watch %v: %v", r.name, r.expectedType, err))
}
// If this is "connection refused" error, it means that most likely apiserver is not responsive.
// It doesn't make sense to re-list all objects because most likely we will be able to restart
// watch where we ended.
// If that's the case wait and resend watch request.
if urlError, ok := err.(*url.Error); ok {
if opError, ok := urlError.Err.(*net.OpError); ok {
if errno, ok := opError.Err.(syscall.Errno); ok && errno == syscall.ECONNREFUSED {
time.Sleep(time.Second)
continue
}
}
}
return nil
}
if err := r.watchHandler(w, &resourceVersion, resyncerrc, stopCh); err != nil {
if err != errorStopRequested {
glog.Warningf("%s: watch of %v ended with: %v", r.name, r.expectedType, err)
}
return nil
}
}
}
// TODO: check for watch expired error and retry watch from latest point? Same issue exists for Until.
func ListWatchUntil(timeout time.Duration, lw ListerWatcher, conditions ...watch.ConditionFunc) (*watch.Event, error) {
if len(conditions) == 0 {
return nil, nil
}
list, err := lw.List(v1.ListOptions{})
if err != nil {
return nil, err
}
initialItems, err := meta.ExtractList(list)
if err != nil {
return nil, err
}
// use the initial items as simulated "adds"
var lastEvent *watch.Event
currIndex := 0
passedConditions := 0
for _, condition := range conditions {
// check the next condition against the previous event and short circuit waiting for the next watch
if lastEvent != nil {
done, err := condition(*lastEvent)
if err != nil {
return lastEvent, err
}
if done {
passedConditions = passedConditions + 1
continue
}
}
ConditionSucceeded:
for currIndex < len(initialItems) {
lastEvent = &watch.Event{Type: watch.Added, Object: initialItems[currIndex]}
currIndex++
done, err := condition(*lastEvent)
if err != nil {
return lastEvent, err
}
if done {
passedConditions = passedConditions + 1
break ConditionSucceeded
}
}
}
if passedConditions == len(conditions) {
return lastEvent, nil
}
remainingConditions := conditions[passedConditions:]
metaObj, err := meta.ListAccessor(list)
if err != nil {
return nil, err
}
currResourceVersion := metaObj.GetResourceVersion()
watchInterface, err := lw.Watch(v1.ListOptions{ResourceVersion: currResourceVersion})
if err != nil {
return nil, err
}
return watch.Until(timeout, watchInterface, remainingConditions...)
}