func (a *app) parseArgs(fs *flag.FlagSet, args []string) error { fs.SetOutput(a.out) fs.Usage = func() { fmt.Fprintf(a.out, help, args[0], args[0]) fs.PrintDefaults() } fs.Var(&a.typeNames, "type", "A generated proto.Message type to generate stubs for (required, repeatable)") fs.StringVar(&a.outFile, "out", "proto_gae.gen.go", "The name of the output file") fs.StringVar(&a.header, "header", copyright, "Header text to put at the top of "+ "the generated file. Defaults to the Chromium Authors copyright.") if err := fs.Parse(args[1:]); err != nil { return err } fail := errors.MultiError(nil) if a.typeNames.Data == nil || a.typeNames.Data.Len() == 0 { fail = append(fail, errors.New("must specify one or more -type")) } if !strings.HasSuffix(a.outFile, ".go") { fail = append(fail, errors.New("-output must end with '.go'")) } if len(fail) > 0 { for _, e := range fail { fmt.Fprintln(a.out, "error:", e) } fmt.Fprintln(a.out) fs.Usage() return fail } return nil }
func (f *fakeDatastore) PutMulti(keys []Key, vals []PropertyMap, cb PutMultiCB) error { if keys[0].Kind() == "FailAll" { return errors.New("PutMulti fail all") } assertExtra := false if _, err := vals[0].GetMeta("assertExtra"); err == nil { assertExtra = true } for i, k := range keys { err := error(nil) if k.Kind() == "Fail" { err = errors.New("PutMulti fail") } else { So(vals[i]["Value"], ShouldResemble, []Property{MkProperty(i)}) if assertExtra { So(vals[i]["Extra"], ShouldResemble, []Property{MkProperty("whoa")}) } if k.Incomplete() { k = mkKey(k.AppID(), k.Namespace(), k.Kind(), int64(i+1), k.Parent()) } } cb(k, err) } return nil }
func (f *fakeDatastore) DeleteMulti(keys []*Key, cb DeleteMultiCB) error { if keys[0].Kind() == "FailAll" { return errors.New("DeleteMulti fail all") } for _, k := range keys { if k.Kind() == "Fail" { cb(errors.New("DeleteMulti fail")) } else { cb(nil) } } return nil }
func (f *fakeDatastore) GetMulti(keys []Key, _meta MultiMetaGetter, cb GetMultiCB) error { if keys[0].Kind() == "FailAll" { return errors.New("GetMulti fail all") } for i, k := range keys { if k.Kind() == "Fail" { cb(nil, errors.New("GetMulti fail")) } else { cb(PropertyMap{"Value": {MkProperty(i + 1)}}, nil) } } return nil }
func (t *taskqueueImpl) deleteLocked(task *tq.Task, queueName string) error { if _, ok := t.archived[queueName][task.Name]; ok { return errors.New("TOMBSTONED_TASK") } if _, ok := t.named[queueName][task.Name]; !ok { return errors.New("UNKNOWN_TASK") } t.archived[queueName][task.Name] = t.named[queueName][task.Name] delete(t.named[queueName], task.Name) return nil }
func (tcf *checkFilter) PutMulti(keys []Key, vals []PropertyMap, cb PutMultiCB) error { if len(keys) != len(vals) { return fmt.Errorf("datastore: PutMulti with mismatched keys/vals lengths (%d/%d)", len(keys), len(vals)) } if len(keys) == 0 { return nil } if cb == nil { return fmt.Errorf("datastore: PutMulti callback is nil") } lme := errors.NewLazyMultiError(len(keys)) for i, k := range keys { if !k.PartialValid(tcf.aid, tcf.ns) { lme.Assign(i, ErrInvalidKey) continue } v := vals[i] if v == nil { lme.Assign(i, errors.New("datastore: PutMulti got nil vals entry")) } } if me := lme.Get(); me != nil { for _, err := range me.(errors.MultiError) { cb(nil, err) } return nil } return tcf.RawInterface.PutMulti(keys, vals, cb) }
func (m *memcacheImpl) Increment(key string, delta int64, initialValue *uint64) (uint64, error) { now := clock.Now(m.ctx) m.data.lock.Lock() defer m.data.lock.Unlock() cur := uint64(0) if initialValue == nil { curItm, err := m.data.retrieveLocked(now, key) if err != nil { return 0, err } if len(curItm.value) != 8 { return 0, errors.New("memcache Increment: got invalid current value") } cur = binary.LittleEndian.Uint64(curItm.value) } else { cur = *initialValue } if delta < 0 { if uint64(-delta) > cur { cur = 0 } else { cur -= uint64(-delta) } } else { cur += uint64(delta) } newval := make([]byte, 8) binary.LittleEndian.PutUint64(newval, cur) m.data.setItemLocked(now, m.NewItem(key).SetValue(newval)) return cur, nil }
// writeMutation ensures that this transaction can support the given key/value // mutation. // // if getOnly is true, don't record the actual mutation data, just ensure that // the key is in an included entity group (or add an empty entry for that // group). // // if !getOnly && data == nil, this counts as a deletion instead of a Put. // // Returns an error if this key causes the transaction to cross too many entity // groups. func (td *txnDataStoreData) writeMutation(getOnly bool, key *ds.Key, data ds.PropertyMap) error { rk := string(keyBytes(key.Root())) td.Lock() defer td.Unlock() if _, ok := td.muts[rk]; !ok { limit := 1 if td.isXG { limit = xgEGLimit } if len(td.muts)+1 > limit { msg := "cross-group transaction need to be explicitly specified (xg=True)" if td.isXG { msg = "operating on too many entity groups in a single transaction" } return errors.New(msg) } td.muts[rk] = []txnMutation{} } if !getOnly { td.muts[rk] = append(td.muts[rk], txnMutation{key, data}) } return nil }
func (t *taskqueueTxnImpl) addLocked(task *tq.Task, queueName string) (*tq.Task, error) { toSched, err := t.parent.prepTask(t.ctx, t.ns, task, queueName) if err != nil { return nil, err } numTasks := 0 for _, vs := range t.anony { numTasks += len(vs) } if numTasks+1 > 5 { // transactional tasks are actually implemented 'for real' as Actions which // ride on the datastore. The current datastore implementation only allows // a maximum of 5 Actions per transaction, and more than that result in a // BAD_REQUEST. return nil, errors.New("BAD_REQUEST") } t.anony[queueName] = append(t.anony[queueName], toSched) // the fact that we have generated a unique name for this task queue item is // an implementation detail. // TODO(riannucci): now that I think about this... it may not actually be true. // We should verify that the .Name for a task added in a transaction is // meaningless. Maybe names generated in a transaction are somehow // guaranteed to be meaningful? toRet := toSched.Duplicate() toRet.Name = "" return toRet, nil }
func (f *fakeDatastore) Run(fq *FinalizedQuery, cb RawRunCB) error { lim, _ := fq.Limit() cursCB := func() (Cursor, error) { return fakeCursor("CURSOR"), nil } for i := int32(0); i < lim; i++ { if v, ok := fq.eqFilts["$err_single"]; ok { idx := fq.eqFilts["$err_single_idx"][0].Value().(int64) if idx == int64(i) { return errors.New(v[0].Value().(string)) } } k := f.mkKey("Kind", i+1) if i == 10 { k = f.mkKey("Kind", "eleven") } pm := PropertyMap{"Value": {MkProperty(i)}} if err := cb(k, pm, cursCB); err != nil { if err == Stop { err = nil } return err } } return nil }
func newPubSubService(ctx context.Context, config pubsubConfig, client *http.Client) (pubSubService, error) { if config.project == "" { return nil, errors.New("pubsub: you must supply a project") } return &pubSubServiceImpl{ ctx: cloud.WithContext(ctx, config.project, client), }, nil }
func (f *FakePLS) Load(pm PropertyMap) error { if f.failLoad { return errors.New("FakePLS.Load") } f.gotLoaded = true f.Value = pm["Value"][0].Value().(int64) return nil }
func (td *txnDataStoreData) run(f func() error) error { // Slightly different from the SDK... datastore and taskqueue each implement // this here, where in the SDK only datastore.transaction.Call does. if atomic.LoadInt32(&td.closed) == 1 { return errors.New("datastore: transaction context has expired") } return f() }
func isOkType(t reflect.Type) error { if t == nil { return errors.New("no type information") } if t.Implements(typeOfPropertyLoadSaver) { return nil } if t == typeOfKey { return errors.New("not user datatype") } if t.Kind() != reflect.Ptr { return errors.New("not a pointer") } if t.Elem().Kind() != reflect.Struct { return errors.New("does not point to a struct") } return nil }
func (d *datastoreImpl) GetAll(q *Query, dst interface{}) error { v := reflect.ValueOf(dst) if v.Kind() != reflect.Ptr { panic(fmt.Errorf("invalid GetAll dst: must have a ptr-to-slice: %T", dst)) } if !v.IsValid() || v.IsNil() { panic(errors.New("invalid GetAll dst: <nil>")) } if keys, ok := dst.(*[]*Key); ok { fq, err := q.KeysOnly(true).Finalize() if err != nil { return err } return d.RawInterface.Run(fq, func(k *Key, _ PropertyMap, _ CursorCB) error { *keys = append(*keys, k) return nil }) } fq, err := q.Finalize() if err != nil { return err } slice := v.Elem() mat := parseMultiArg(slice.Type()) if mat.newElem == nil { panic(fmt.Errorf("invalid GetAll dst (non-concrete element type): %T", dst)) } errs := map[int]error{} i := 0 err = d.RawInterface.Run(fq, func(k *Key, pm PropertyMap, _ CursorCB) error { slice.Set(reflect.Append(slice, mat.newElem())) itm := slice.Index(i) mat.setKey(itm, k) err := mat.setPM(itm, pm) if err != nil { errs[i] = err } i++ return nil }) if err == nil { if len(errs) > 0 { me := make(errors.MultiError, slice.Len()) for i, e := range errs { me[i] = e } err = me } } return err }
func (s *testPubSubService) Pull(sub string, count int) (msgs []*pubsub.Message, err error) { mock, e := s.PopErr("Pull", sub, count) if e != nil { if s.infinitePull { return nil, nil } s.AddError(errors.New("out of mock Pull entries")) } mock.BindResult(&msgs, &err) return }
func (d *dataStoreData) allocateIDsLocked(ents *memCollection, incomplete *ds.Key, n int) (int64, error) { if d.disableSpecialEntities { return 0, errors.New("disableSpecialEntities is true so allocateIDs is disabled") } idKey := []byte(nil) if incomplete.Parent() == nil { idKey = rootIDsKey(incomplete.Kind()) } else { idKey = groupIDsKey(incomplete) } return incrementLocked(ents, idKey, n), nil }
func TestBrokenFeatures(t *testing.T) { t.Parallel() e := errors.New("default err") Convey("BrokenFeatures", t, func() { c := memory.Use(context.Background()) Convey("Can break ds", func() { Convey("without a default", func() { c, bf := FilterRDS(c, nil) ds := datastore.Get(c) vals := []datastore.PropertyMap{{ "$key": {datastore.MkPropertyNI(ds.NewKey("Wut", "", 1, nil))}, }} Convey("by specifying an error", func() { bf.BreakFeatures(e, "GetMulti", "PutMulti") So(ds.GetMulti(vals), ShouldEqual, e) Convey("and you can unbreak them as well", func() { bf.UnbreakFeatures("GetMulti") So(errors.SingleError(ds.GetMulti(vals)), ShouldEqual, datastore.ErrNoSuchEntity) Convey("no broken features at all is a shortcut", func() { bf.UnbreakFeatures("PutMulti") So(errors.SingleError(ds.GetMulti(vals)), ShouldEqual, datastore.ErrNoSuchEntity) }) }) }) Convey("Not specifying an error gets you a generic error", func() { bf.BreakFeatures(nil, "GetMulti") So(ds.GetMulti(vals).Error(), ShouldContainSubstring, `feature "GetMulti" is broken`) }) }) Convey("with a default", func() { c, bf := FilterRDS(c, e) ds := datastore.Get(c) vals := []datastore.PropertyMap{{ "$key": {datastore.MkPropertyNI(ds.NewKey("Wut", "", 1, nil))}, }} bf.BreakFeatures(nil, "GetMulti") So(ds.GetMulti(vals), ShouldEqual, e) }) }) }) }
func (c *endpointConfig) createService(ctx context.Context) (endpointService, error) { if c.url == "" { return nil, errors.New("endpoint: you must supply a monitoring endpoint") } authenticator := auth.NewAuthenticator( auth.Options{ Method: auth.ServiceAccountMethod, Scopes: endpointScopes, ServiceAccountJSONPath: c.serviceAccountJSONPath, Logger: log.Get(ctx), }) client, err := auth.AuthenticatedClient(auth.SilentLogin, authenticator) if err != nil { log.Errorf(log.SetError(ctx, err), "Failed to configure endpoint client.") return nil, errors.New("endpoint: failed to configure endpoint client") } return &endpointServiceImpl{ endpointConfig: *c, client: client, }, nil }
// newPubSubClient instantiates a new Pub/Sub client. // // This method will also perform authentication and setup the topic/subscription // if it isn't already set up. func newPubSubClient(ctx context.Context, config pubsubConfig, svc pubSubService) (*pubsubClient, error) { if config.subscription == "" { return nil, errors.New("pubsub: you must supply a subscription") } if config.batchSize <= 0 { return nil, errors.New("pubsub: batch size must be at least 1") } else if config.batchSize > maxSubscriptionPullSize { return nil, fmt.Errorf("pubsub: batch size cannot exceed %d", maxSubscriptionPullSize) } p := pubsubClient{ pubsubConfig: &config, service: svc, } // Ensure that our Subscription (and topic) exist. if err := p.setupSubscription(ctx); err != nil { log.Errorf(log.SetError(ctx, err), "Failed to set up subscription.") return nil, err } return &p, nil }
func (f *FakePLS) GetMeta(key string) (interface{}, error) { if f.failGetMeta { return nil, errors.New("FakePLS.GetMeta") } switch key { case "id": if f.StringID != "" { return f.StringID, nil } return f.IntID, nil case "kind": if f.Kind == "" { return "FakePLS", nil } return f.Kind, nil } return nil, ErrMetaFieldUnset }
func (t *taskqueueTxnImpl) AddMulti(tasks []*tq.Task, queueName string, cb tq.RawTaskCB) error { if atomic.LoadInt32(&t.closed) == 1 { return errors.New("taskqueue: transaction context has expired") } t.Lock() defer t.Unlock() queueName, err := t.parent.getQueueNameLocked(queueName) if err != nil { return err } for _, task := range tasks { cb(t.addLocked(task, queueName)) } return nil }
func (f *FakePLS) SetMeta(key string, val interface{}) error { if f.failSetMeta { return errors.New("FakePL.SetMeta") } if key == "id" { switch x := val.(type) { case int64: f.IntID = x case string: f.StringID = x } return nil } if key == "kind" { f.Kind = val.(string) return nil } return ErrMetaFieldUnset }
func (d *dsTxnBuf) Run(fq *ds.FinalizedQuery, cb ds.RawRunCB) error { if start, end := fq.Bounds(); start != nil || end != nil { return errors.New("txnBuf filter does not support query cursors") } limit, limitSet := fq.Limit() offset, _ := fq.Offset() keysOnly := fq.KeysOnly() project := fq.Project() bufDS, parentDS, sizes := func() (ds.RawInterface, ds.RawInterface, *sizeTracker) { if !d.haveLock { d.state.Lock() defer d.state.Unlock() } return d.state.bufDS, d.state.parentDS, d.state.entState.dup() }() return runMergedQueries(fq, sizes, bufDS, parentDS, func(key *ds.Key, data ds.PropertyMap) error { if offset > 0 { offset-- return nil } if limitSet { if limit == 0 { return ds.Stop } limit-- } if keysOnly { data = nil } else if len(project) > 0 { newData := make(ds.PropertyMap, len(project)) for _, p := range project { newData[p] = data[p] } data = newData } return cb(key, data, nil) }) }
func (f *FakePLS) Save(withMeta bool) (PropertyMap, error) { if f.failSave { return nil, errors.New("FakePLS.Save") } ret := PropertyMap{ "Value": {MkProperty(f.Value)}, "Extra": {MkProperty("whoa")}, } if withMeta { id, _ := f.GetMeta("id") So(ret.SetMeta("id", id), ShouldBeTrue) if f.Kind == "" { So(ret.SetMeta("kind", "FakePLS"), ShouldBeTrue) } else { So(ret.SetMeta("kind", f.Kind), ShouldBeTrue) } So(ret.SetMeta("assertExtra", true), ShouldBeTrue) } return ret, nil }
func (d *datastoreImpl) GetAll(q Query, dst interface{}) error { v := reflect.ValueOf(dst) if v.Kind() != reflect.Ptr { return fmt.Errorf("invalid GetAll dst: must have a ptr-to-slice: %T", dst) } if !v.IsValid() || v.IsNil() { return errors.New("invalid GetAll dst: <nil>") } if keys, ok := dst.(*[]Key); ok { return d.RawInterface.Run(q.KeysOnly(), func(k Key, _ PropertyMap, _ CursorCB) bool { *keys = append(*keys, k) return true }) } slice := v.Elem() mat := parseMultiArg(slice.Type()) if !mat.valid || mat.newElem == nil { return fmt.Errorf("invalid GetAll input type: %T", dst) } lme := errors.NewLazyMultiError(slice.Len()) i := 0 err := d.RawInterface.Run(q, func(k Key, pm PropertyMap, _ CursorCB) bool { slice.Set(reflect.Append(slice, mat.newElem())) itm := slice.Index(i) mat.setKey(itm, k) lme.Assign(i, mat.setPM(itm, pm)) i++ return true }) if err == nil { err = lme.Get() } return err }
func (p *structPLS) save(propMap PropertyMap, prefix string, is IndexSetting) (idxCount int, err error) { saveProp := func(name string, si IndexSetting, v reflect.Value, st *structTag) (err error) { if st.substructCodec != nil { count, err := (&structPLS{v, st.substructCodec}).save(propMap, name, si) if err == nil { idxCount += count if idxCount > maxIndexedProperties { err = errors.New("gae: too many indexed properties") } } return err } prop := Property{} if st.convert { prop, err = v.Addr().Interface().(PropertyConverter).ToProperty() } else { err = prop.SetValue(v.Interface(), si) } if err != nil { return err } propMap[name] = append(propMap[name], prop) if prop.IndexSetting() == ShouldIndex { idxCount++ if idxCount > maxIndexedProperties { return errors.New("gae: too many indexed properties") } } return nil } for i, st := range p.c.byIndex { if st.name == "-" || st.isExtra { continue } name := st.name if prefix != "" { name = prefix + name } v := p.o.Field(i) is1 := is if st.idxSetting == NoIndex { is1 = NoIndex } if st.isSlice { for j := 0; j < v.Len(); j++ { if err = saveProp(name, is1, v.Index(j), &st); err != nil { return } } } else { if err = saveProp(name, is1, v, &st); err != nil { return } } } if i, ok := p.c.bySpecial["extra"]; ok { if p.c.byIndex[i].name != "-" { for fullName, vals := range p.o.Field(i).Interface().(PropertyMap) { if _, ok := propMap[fullName]; !ok { propMap[fullName] = vals } } } } return }
func (t *taskqueueTxnImpl) Stats([]string, tq.RawStatsCB) error { return errors.New("taskqueue: cannot Stats from a transaction") }
) const ( // The maximum number of items that can be pulled from a subscription at once. maxSubscriptionPullSize = 100 ) var ( // OAuth2 scopes to generate. pubsubScopes = []string{ pubsub.ScopePubSub, auth.OAuthScopeEmail, } // Error returned by pullAckMessages to indicate that no messages were available. errNoMessages = errors.New("pubsub: no messages") ) // pubsubConfig is the set of configuration parameters for a pubsubClient. type pubsubConfig struct { project string // The project name. topic string // The topic name. subscription string // The subscription name. create bool batchSize int // The number of elements to pull from a subscription per batch. } // addFlags adds this configuration's set of flags to a FlagSet. func (c *pubsubConfig) addFlags(fs *flag.FlagSet) { fs.StringVar(&c.project, "pubsub-project", "", "The name of the Pub/Sub project.") fs.StringVar(&c.subscription, "pubsub-subscription", "", "The name of the Pub/Sub subscription.")
// setupSubscription asserts that the configured subscription exists. In doing // so, it also asserts that the client credentials are valid with respect to the // configured project/subscription. // // If the subscription doesn't exist, this method can create the subscription // and (if missing) its topic, if the "create" flag is set. func (p *pubsubClient) setupSubscription(ctx context.Context) error { exists := false log.Fields{ "subscription": p.topic, }.Infof(ctx, "Checking for subscription existence.") err := retryCall(ctx, "SubExists()", func() error { var err error exists, err = p.service.SubExists(p.subscription) return p.wrapTransient(err) }) if err != nil { log.Warningf(log.SetError(ctx, err), "Failed to test for subscription; assuming it doesn't exist.") } if exists { return nil } if !p.create { return errors.New("pubsub: subscription doesn't exist, not configured to create") } // Create the subscription if it doesn't exist. if p.topic == "" { log.Errorf(ctx, "Cannot create subscription; no topic was specified.") return errors.New("pubsub: cannot create subscription") } // Test if the topic exists... log.Fields{ "topic": p.topic, }.Infof(ctx, "Checking for topic existence.") err = retryCall(ctx, "TopicExists()", func() error { var err error exists, err = p.service.TopicExists(p.topic) return p.wrapTransient(err) }) if err != nil { log.Warningf(log.SetError(ctx, err), "Failed to test for topic; assuming it doesn't exist.") } if !exists { log.Fields{ "topic": p.topic, }.Infof(ctx, "Creating topic.") err := retryCall(ctx, "CreateTopic()", func() error { return p.service.CreateTopic(p.topic) }) if err != nil { log.Warningf(log.SetError(ctx, err), "Failed to create topic.") return errors.New("pubsub: cannot create topic") } } log.Fields{ "topic": p.topic, "subscription": p.subscription, }.Infof(ctx, "Creating pull subscription for topic.") if err := retryCall(ctx, "CreateSub()", func() error { return p.service.CreatePullSub(p.subscription, p.topic) }); err != nil { log.Warningf(log.SetError(ctx, err), "Failed to test for subscription; assuming it doesn't exist.") return errors.New("pubsub: failed to create subscription") } return nil }