func (s *SortingPrinter) sortObj(obj runtime.Object) error { objs, err := meta.ExtractList(obj) if err != nil { return err } if len(objs) == 0 { return nil } sorter, err := SortObjects(s.Decoder, objs, s.SortField) if err != nil { return err } switch list := obj.(type) { case *v1.List: outputList := make([]runtime.RawExtension, len(objs)) for ix := range objs { outputList[ix] = list.Items[sorter.OriginalPosition(ix)] } list.Items = outputList return nil } return meta.SetList(obj, objs) }
func (s *CustomColumnsPrinter) PrintObj(obj runtime.Object, out io.Writer) error { w := tabwriter.NewWriter(out, columnwidth, tabwidth, padding, padding_character, flags) if !s.NoHeaders { headers := make([]string, len(s.Columns)) for ix := range s.Columns { headers[ix] = s.Columns[ix].Header } fmt.Fprintln(w, strings.Join(headers, "\t")) } parsers := make([]*jsonpath.JSONPath, len(s.Columns)) for ix := range s.Columns { parsers[ix] = jsonpath.New(fmt.Sprintf("column%d", ix)) if err := parsers[ix].Parse(s.Columns[ix].FieldSpec); err != nil { return err } } if meta.IsListType(obj) { objs, err := meta.ExtractList(obj) if err != nil { return err } for ix := range objs { if err := s.printOneObject(objs[ix], parsers, w); err != nil { return err } } } else { if err := s.printOneObject(obj, parsers, w); err != nil { return err } } return w.Flush() }
func TestGetMultipleTypeObjectsAsList(t *testing.T) { pods, svc, _ := testData() f, tf, codec, _ := cmdtesting.NewAPIFactory() tf.Printer = &testPrinter{} tf.Client = &fake.RESTClient{ NegotiatedSerializer: unstructuredSerializer, Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { switch req.URL.Path { case "/namespaces/test/pods": return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, pods)}, nil case "/namespaces/test/services": return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, svc)}, nil default: t.Fatalf("unexpected request: %#v\n%#v", req.URL, req) return nil, nil } }), } tf.Namespace = "test" tf.ClientConfig = &restclient.Config{ContentConfig: restclient.ContentConfig{GroupVersion: &api.Registry.GroupOrDie(api.GroupName).GroupVersion}} buf := bytes.NewBuffer([]byte{}) errBuf := bytes.NewBuffer([]byte{}) cmd := NewCmdGet(f, buf, errBuf) cmd.SetOutput(buf) cmd.Flags().Set("output", "json") cmd.Run(cmd, []string{"pods,services"}) if tf.Printer.(*testPrinter).Objects != nil { t.Errorf("unexpected print to default printer") } out, err := runtime.Decode(codec, buf.Bytes()) if err != nil { t.Fatalf("unexpected error: %v", err) } list, err := meta.ExtractList(out) if err != nil { t.Fatalf("unexpected error: %v", err) } if errs := runtime.DecodeList(list, codec); len(errs) > 0 { t.Fatalf("unexpected error: %v", errs) } if err := meta.SetList(out, list); err != nil { t.Fatalf("unexpected error: %v", err) } expected := &api.List{ Items: []runtime.Object{ &pods.Items[0], &pods.Items[1], &svc.Items[0], }, } if !reflect.DeepEqual(expected, out) { t.Errorf("unexpected output: %#v", out) } }
// CreateList will properly create a list using the storage interface func CreateList(prefix string, helper storage.Interface, list runtime.Object) error { items, err := meta.ExtractList(list) if err != nil { return err } err = CreateObjList(prefix, helper, items) if err != nil { return err } return meta.SetList(list, items) }
func TestExtractListOfInterfacePtrs(t *testing.T) { pl := &fakePtrInterfaceList{ Items: &[]runtime.Object{}, } list, err := meta.ExtractList(pl) if err != nil { t.Fatalf("Unexpected error %v", err) } if len(list) > 0 { t.Fatalf("Expected empty list, got %#v", list) } }
func extractResourceList(objs []runtime.Object) ([]runtime.Object, error) { finalObjs := []runtime.Object{} for _, obj := range objs { items, err := meta.ExtractList(obj) if err != nil { return nil, err } for _, item := range items { finalObjs = append(finalObjs, item) } } return finalObjs, nil }
func verifyListMetadata(t *testing.T, metaOnlyList *MetadataOnlyObjectList) { items, err := meta.ExtractList(metaOnlyList) if err != nil { t.Fatal(err) } for _, item := range items { metaOnly, ok := item.(*MetadataOnlyObject) if !ok { t.Fatalf("expected item to be *MetadataOnlyObject") } verfiyMetadata("check list", t, metaOnly) } }
func (p *pruner) prune(namespace string, mapping *meta.RESTMapping, shortOutput bool) error { c, err := p.clientFunc(mapping) if err != nil { return err } objList, err := resource.NewHelper(c, mapping).List(namespace, mapping.GroupVersionKind.Version, p.selector, false) if err != nil { return err } objs, err := meta.ExtractList(objList) if err != nil { return err } for _, obj := range objs { annots, err := mapping.MetadataAccessor.Annotations(obj) if err != nil { return err } if _, ok := annots[annotations.LastAppliedConfigAnnotation]; !ok { // don't prune resources not created with apply continue } uid, err := mapping.UID(obj) if err != nil { return err } if p.visitedUids.Has(string(uid)) { continue } name, err := mapping.Name(obj) if err != nil { return err } if !p.dryRun { if err := p.delete(namespace, name, mapping, c); err != nil { return err } } cmdutil.PrintSuccess(p.mapper, shortOutput, p.out, mapping.Resource, name, p.dryRun, "pruned") } return nil }
func (t *tracker) addList(obj runtime.Object, replaceExisting bool) error { list, err := meta.ExtractList(obj) if err != nil { return err } errs := runtime.DecodeList(list, t.decoder) if len(errs) > 0 { return errs[0] } for _, obj := range list { objMeta, err := meta.Accessor(obj) if err != nil { return err } err = t.add(obj, objMeta.GetNamespace(), replaceExisting) if err != nil { return err } } return nil }
func (v FlattenListVisitor) Visit(fn VisitorFunc) error { return v.Visitor.Visit(func(info *Info, err error) error { if err != nil { return err } if info.Object == nil { return fn(info, nil) } items, err := meta.ExtractList(info.Object) if err != nil { return fn(info, nil) } if errs := runtime.DecodeList(items, struct { runtime.ObjectTyper runtime.Decoder }{v.Mapper, v.Mapper.Decoder}); len(errs) > 0 { return utilerrors.NewAggregate(errs) } // If we have a GroupVersionKind on the list, prioritize that when asking for info on the objects contained in the list var preferredGVKs []schema.GroupVersionKind if info.Mapping != nil && !info.Mapping.GroupVersionKind.Empty() { preferredGVKs = append(preferredGVKs, info.Mapping.GroupVersionKind) } for i := range items { item, err := v.InfoForObject(items[i], preferredGVKs) if err != nil { return err } if len(info.ResourceVersion) != 0 { item.ResourceVersion = info.ResourceVersion } if err := fn(item, nil); err != nil { return err } } return nil }) }
func TestSetExtractListRoundTrip(t *testing.T) { fuzzer := fuzz.New().NilChance(0).NumElements(1, 5) for i := 0; i < 5; i++ { start := &api.PodList{} fuzzer.Fuzz(&start.Items) list, err := meta.ExtractList(start) if err != nil { t.Errorf("Unexpected error %v", err) continue } got := &api.PodList{} err = meta.SetList(got, list) if err != nil { t.Errorf("Unexpected error %v", err) continue } if e, a := start, got; !reflect.DeepEqual(e, a) { t.Fatalf("Expected %#v, got %#v", e, a) } } }
// FilterResourceList receives a list of runtime objects. // If any objects are filtered, that number is returned along with a modified list. func FilterResourceList(obj runtime.Object, filterFuncs kubectl.Filters, filterOpts *kubectl.PrintOptions) (int, []runtime.Object, error) { items, err := meta.ExtractList(obj) if err != nil { return 0, []runtime.Object{obj}, utilerrors.NewAggregate([]error{err}) } if errs := runtime.DecodeList(items, api.Codecs.UniversalDecoder(), unstructured.UnstructuredJSONScheme); len(errs) > 0 { return 0, []runtime.Object{obj}, utilerrors.NewAggregate(errs) } filterCount := 0 list := make([]runtime.Object, 0, len(items)) for _, obj := range items { if isFiltered, err := filterFuncs.Filter(obj, filterOpts); !isFiltered { if err != nil { glog.V(2).Infof("Unable to filter resource: %v", err) continue } list = append(list, obj) } else if isFiltered { filterCount++ } } return filterCount, list, nil }
// ListAndWatch first lists all items and get the resource version at the moment of call, // and then use the resource version to watch. // It returns error if ListAndWatch didn't even try to initialize watch. func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error { glog.V(3).Infof("Listing and watching %v from %s", r.expectedType, r.name) var resourceVersion string resyncCh, cleanup := r.resyncChan() defer cleanup() // Explicitly set "0" as resource version - it's fine for the List() // to be served from cache and potentially be delayed relative to // etcd contents. Reflector framework will catch up via Watch() eventually. options := v1.ListOptions{ResourceVersion: "0"} list, err := r.listerWatcher.List(options) if err != nil { return fmt.Errorf("%s: Failed to list %v: %v", r.name, r.expectedType, err) } listMetaInterface, err := meta.ListAccessor(list) if err != nil { return fmt.Errorf("%s: Unable to understand list result %#v: %v", r.name, list, err) } resourceVersion = listMetaInterface.GetResourceVersion() items, err := meta.ExtractList(list) if err != nil { return fmt.Errorf("%s: Unable to understand list result %#v (%v)", r.name, list, err) } if err := r.syncWith(items, resourceVersion); err != nil { return fmt.Errorf("%s: Unable to sync list result: %v", r.name, err) } r.setLastSyncResourceVersion(resourceVersion) resyncerrc := make(chan error, 1) cancelCh := make(chan struct{}) defer close(cancelCh) go func() { for { select { case <-resyncCh: case <-stopCh: return case <-cancelCh: return } glog.V(4).Infof("%s: forcing resync", r.name) if err := r.store.Resync(); err != nil { resyncerrc <- err return } cleanup() resyncCh, cleanup = r.resyncChan() } }() for { timemoutseconds := int64(minWatchTimeout.Seconds() * (rand.Float64() + 1.0)) options = v1.ListOptions{ ResourceVersion: resourceVersion, // We want to avoid situations of hanging watchers. Stop any wachers that do not // receive any events within the timeout window. TimeoutSeconds: &timemoutseconds, } w, err := r.listerWatcher.Watch(options) if err != nil { switch err { case io.EOF: // watch closed normally case io.ErrUnexpectedEOF: glog.V(1).Infof("%s: Watch for %v closed with unexpected EOF: %v", r.name, r.expectedType, err) default: utilruntime.HandleError(fmt.Errorf("%s: Failed to watch %v: %v", r.name, r.expectedType, err)) } // If this is "connection refused" error, it means that most likely apiserver is not responsive. // It doesn't make sense to re-list all objects because most likely we will be able to restart // watch where we ended. // If that's the case wait and resend watch request. if urlError, ok := err.(*url.Error); ok { if opError, ok := urlError.Err.(*net.OpError); ok { if errno, ok := opError.Err.(syscall.Errno); ok && errno == syscall.ECONNREFUSED { time.Sleep(time.Second) continue } } } return nil } if err := r.watchHandler(w, &resourceVersion, resyncerrc, stopCh); err != nil { if err != errorStopRequested { glog.Warningf("%s: watch of %v ended with: %v", r.name, r.expectedType, err) } return nil } } }
// DeleteCollection remove all items returned by List with a given ListOptions from storage. // // DeleteCollection is currently NOT atomic. It can happen that only subset of objects // will be deleted from storage, and then an error will be returned. // In case of success, the list of deleted objects will be returned. // // TODO: Currently, there is no easy way to remove 'directory' entry from storage (if we // are removing all objects of a given type) with the current API (it's technically // possibly with storage API, but watch is not delivered correctly then). // It will be possible to fix it with v3 etcd API. func (e *Store) DeleteCollection(ctx genericapirequest.Context, options *api.DeleteOptions, listOptions *api.ListOptions) (runtime.Object, error) { listObj, err := e.List(ctx, listOptions) if err != nil { return nil, err } items, err := meta.ExtractList(listObj) if err != nil { return nil, err } // Spawn a number of goroutines, so that we can issue requests to storage // in parallel to speed up deletion. // TODO: Make this proportional to the number of items to delete, up to // DeleteCollectionWorkers (it doesn't make much sense to spawn 16 // workers to delete 10 items). workersNumber := e.DeleteCollectionWorkers if workersNumber < 1 { workersNumber = 1 } wg := sync.WaitGroup{} toProcess := make(chan int, 2*workersNumber) errs := make(chan error, workersNumber+1) go func() { defer utilruntime.HandleCrash(func(panicReason interface{}) { errs <- fmt.Errorf("DeleteCollection distributor panicked: %v", panicReason) }) for i := 0; i < len(items); i++ { toProcess <- i } close(toProcess) }() wg.Add(workersNumber) for i := 0; i < workersNumber; i++ { go func() { // panics don't cross goroutine boundaries defer utilruntime.HandleCrash(func(panicReason interface{}) { errs <- fmt.Errorf("DeleteCollection goroutine panicked: %v", panicReason) }) defer wg.Done() for { index, ok := <-toProcess if !ok { return } accessor, err := meta.Accessor(items[index]) if err != nil { errs <- err return } if _, err := e.Delete(ctx, accessor.GetName(), options); err != nil && !kubeerr.IsNotFound(err) { glog.V(4).Infof("Delete %s in DeleteCollection failed: %v", accessor.GetName(), err) errs <- err return } } }() } wg.Wait() select { case err := <-errs: return nil, err default: return listObj, nil } }
func TestExtractList(t *testing.T) { list1 := []runtime.Object{ &api.Pod{ObjectMeta: metav1.ObjectMeta{Name: "1"}}, &api.Service{ObjectMeta: metav1.ObjectMeta{Name: "2"}}, } list2 := &v1.List{ Items: []runtime.RawExtension{ {Raw: []byte("foo")}, {Raw: []byte("bar")}, {Object: &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "other"}}}, }, } list3 := &fakePtrValueList{ Items: []*api.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "2"}}, }, } list4 := &api.PodList{ Items: []api.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "2"}}, {ObjectMeta: metav1.ObjectMeta{Name: "3"}}, }, } list5 := &v1.PodList{ Items: []v1.Pod{ {ObjectMeta: metav1.ObjectMeta{Name: "1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "2"}}, {ObjectMeta: metav1.ObjectMeta{Name: "3"}}, }, } testCases := []struct { in runtime.Object out []interface{} equal bool }{ { in: &api.List{}, out: []interface{}{}, }, { in: &v1.List{}, out: []interface{}{}, }, { in: &v1.PodList{}, out: []interface{}{}, }, { in: &api.List{Items: list1}, out: []interface{}{list1[0], list1[1]}, }, { in: list2, out: []interface{}{&runtime.Unknown{Raw: list2.Items[0].Raw}, &runtime.Unknown{Raw: list2.Items[1].Raw}, list2.Items[2].Object}, equal: true, }, { in: list3, out: []interface{}{list3.Items[0], list3.Items[1]}, }, { in: list4, out: []interface{}{&list4.Items[0], &list4.Items[1], &list4.Items[2]}, }, { in: list5, out: []interface{}{&list5.Items[0], &list5.Items[1], &list5.Items[2]}, }, } for i, test := range testCases { list, err := meta.ExtractList(test.in) if err != nil { t.Fatalf("%d: extract: Unexpected error %v", i, err) } if e, a := len(test.out), len(list); e != a { t.Fatalf("%d: extract: Expected %v, got %v", i, e, a) } for j, e := range test.out { if e != list[j] { if !test.equal { t.Fatalf("%d: extract: Expected list[%d] to be %#v, but found %#v", i, j, e, list[j]) } if !reflect.DeepEqual(e, list[j]) { t.Fatalf("%d: extract: Expected list[%d] to be %#v, but found %#v", i, j, e, list[j]) } } } } }
func TestArrayOfRuntimeObject(t *testing.T) { internalGV := schema.GroupVersion{Group: "test.group", Version: runtime.APIVersionInternal} externalGV := schema.GroupVersion{Group: "test.group", Version: "v1test"} s := runtime.NewScheme() s.AddKnownTypes(internalGV, &EmbeddedTest{}) s.AddKnownTypeWithName(externalGV.WithKind("EmbeddedTest"), &EmbeddedTestExternal{}) s.AddKnownTypes(internalGV, &ObjectTest{}) s.AddKnownTypeWithName(externalGV.WithKind("ObjectTest"), &ObjectTestExternal{}) codec := serializer.NewCodecFactory(s).LegacyCodec(externalGV) innerItems := []runtime.Object{ &EmbeddedTest{ID: "baz"}, } items := []runtime.Object{ &EmbeddedTest{ID: "foo"}, &EmbeddedTest{ID: "bar"}, // TODO: until YAML is removed, this JSON must be in ascending key order to ensure consistent roundtrip serialization &runtime.Unknown{ Raw: []byte(`{"apiVersion":"unknown.group/unknown","foo":"bar","kind":"OtherTest"}`), ContentType: runtime.ContentTypeJSON, }, &ObjectTest{ Items: runtime.NewEncodableList(codec, innerItems), }, } internal := &ObjectTest{ Items: runtime.NewEncodableList(codec, items), } wire, err := runtime.Encode(codec, internal) if err != nil { t.Fatalf("unexpected error: %v", err) } t.Logf("Wire format is:\n%s\n", string(wire)) obj := &ObjectTestExternal{} if err := json.Unmarshal(wire, obj); err != nil { t.Fatalf("unexpected error: %v", err) } t.Logf("exact wire is: %s", string(obj.Items[0].Raw)) items[3] = &ObjectTest{Items: innerItems} internal.Items = items decoded, err := runtime.Decode(codec, wire) if err != nil { t.Fatalf("unexpected error: %v", err) } list, err := meta.ExtractList(decoded) if err != nil { t.Fatalf("unexpected error: %v", err) } if errs := runtime.DecodeList(list, codec); len(errs) > 0 { t.Fatalf("unexpected error: %v", errs) } list2, err := meta.ExtractList(list[3]) if err != nil { t.Fatalf("unexpected error: %v", err) } if errs := runtime.DecodeList(list2, codec); len(errs) > 0 { t.Fatalf("unexpected error: %v", errs) } if err := meta.SetList(list[3], list2); err != nil { t.Fatalf("unexpected error: %v", err) } // we want DecodeList to set type meta if possible, even on runtime.Unknown objects internal.Items[2].(*runtime.Unknown).TypeMeta = runtime.TypeMeta{Kind: "OtherTest", APIVersion: "unknown.group/unknown"} if e, a := internal.Items, list; !reflect.DeepEqual(e, a) { t.Errorf("mismatched decoded: %s", diff.ObjectGoPrintSideBySide(e, a)) } }
// TODO: check for watch expired error and retry watch from latest point? Same issue exists for Until. func ListWatchUntil(timeout time.Duration, lw ListerWatcher, conditions ...watch.ConditionFunc) (*watch.Event, error) { if len(conditions) == 0 { return nil, nil } list, err := lw.List(v1.ListOptions{}) if err != nil { return nil, err } initialItems, err := meta.ExtractList(list) if err != nil { return nil, err } // use the initial items as simulated "adds" var lastEvent *watch.Event currIndex := 0 passedConditions := 0 for _, condition := range conditions { // check the next condition against the previous event and short circuit waiting for the next watch if lastEvent != nil { done, err := condition(*lastEvent) if err != nil { return lastEvent, err } if done { passedConditions = passedConditions + 1 continue } } ConditionSucceeded: for currIndex < len(initialItems) { lastEvent = &watch.Event{Type: watch.Added, Object: initialItems[currIndex]} currIndex++ done, err := condition(*lastEvent) if err != nil { return lastEvent, err } if done { passedConditions = passedConditions + 1 break ConditionSucceeded } } } if passedConditions == len(conditions) { return lastEvent, nil } remainingConditions := conditions[passedConditions:] metaObj, err := meta.ListAccessor(list) if err != nil { return nil, err } currResourceVersion := metaObj.GetResourceVersion() watchInterface, err := lw.Watch(v1.ListOptions{ResourceVersion: currResourceVersion}) if err != nil { return nil, err } return watch.Until(timeout, watchInterface, remainingConditions...) }
func visitToPatch( originalObj runtime.Object, updates *resource.Info, mapper meta.RESTMapper, resourceMapper *resource.Mapper, encoder runtime.Encoder, out, errOut io.Writer, defaultVersion schema.GroupVersion, results *editResults, file string, ) error { patchVisitor := resource.NewFlattenListVisitor(updates, resourceMapper) err := patchVisitor.Visit(func(info *resource.Info, incomingErr error) error { currOriginalObj := originalObj // if we're editing a list, then navigate the list to find the item that we're currently trying to edit if meta.IsListType(originalObj) { currOriginalObj = nil editObjUID, err := meta.NewAccessor().UID(info.Object) if err != nil { return err } listItems, err := meta.ExtractList(originalObj) if err != nil { return err } // iterate through the list to find the item with the matching UID for i := range listItems { originalObjUID, err := meta.NewAccessor().UID(listItems[i]) if err != nil { return err } if editObjUID == originalObjUID { currOriginalObj = listItems[i] break } } if currOriginalObj == nil { return fmt.Errorf("no original object found for %#v", info.Object) } } originalSerialization, err := runtime.Encode(encoder, currOriginalObj) if err != nil { return err } editedSerialization, err := runtime.Encode(encoder, info.Object) if err != nil { return err } // compute the patch on a per-item basis // use strategic merge to create a patch originalJS, err := yaml.ToJSON(originalSerialization) if err != nil { return err } editedJS, err := yaml.ToJSON(editedSerialization) if err != nil { return err } if reflect.DeepEqual(originalJS, editedJS) { // no edit, so just skip it. cmdutil.PrintSuccess(mapper, false, out, info.Mapping.Resource, info.Name, false, "skipped") return nil } preconditions := []strategicpatch.PreconditionFunc{strategicpatch.RequireKeyUnchanged("apiVersion"), strategicpatch.RequireKeyUnchanged("kind"), strategicpatch.RequireMetadataKeyUnchanged("name")} patch, err := strategicpatch.CreateTwoWayMergePatch(originalJS, editedJS, currOriginalObj, preconditions...) if err != nil { glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) if strategicpatch.IsPreconditionFailed(err) { return fmt.Errorf("%s", "At least one of apiVersion, kind and name was changed") } return err } results.version = defaultVersion patched, err := resource.NewHelper(info.Client, info.Mapping).Patch(info.Namespace, info.Name, types.StrategicMergePatchType, patch) if err != nil { fmt.Fprintln(out, results.addError(err, info)) return nil } info.Refresh(patched, true) cmdutil.PrintSuccess(mapper, false, out, info.Mapping.Resource, info.Name, false, "edited") return nil }) return err }