func (a *app) parseArgs(fs *flag.FlagSet, args []string) error {
fs.SetOutput(a.out)
fs.Usage = func() {
fmt.Fprintf(a.out, help, args[0], args[0])
fs.PrintDefaults()
}
fs.Var(&a.typeNames, "type",
"A generated proto.Message type to generate stubs for (required, repeatable)")
fs.StringVar(&a.outFile, "out", "proto_gae.gen.go",
"The name of the output file")
fs.StringVar(&a.header, "header", copyright, "Header text to put at the top of "+
"the generated file. Defaults to the Chromium Authors copyright.")
if err := fs.Parse(args[1:]); err != nil {
return err
}
fail := errors.MultiError(nil)
if a.typeNames.Data == nil || a.typeNames.Data.Len() == 0 {
fail = append(fail, errors.New("must specify one or more -type"))
}
if !strings.HasSuffix(a.outFile, ".go") {
fail = append(fail, errors.New("-output must end with '.go'"))
}
if len(fail) > 0 {
for _, e := range fail {
fmt.Fprintln(a.out, "error:", e)
}
fmt.Fprintln(a.out)
fs.Usage()
return fail
}
return nil
}
func (f *fakeDatastore) PutMulti(keys []Key, vals []PropertyMap, cb PutMultiCB) error {
if keys[0].Kind() == "FailAll" {
return errors.New("PutMulti fail all")
}
assertExtra := false
if _, err := vals[0].GetMeta("assertExtra"); err == nil {
assertExtra = true
}
for i, k := range keys {
err := error(nil)
if k.Kind() == "Fail" {
err = errors.New("PutMulti fail")
} else {
So(vals[i]["Value"], ShouldResemble, []Property{MkProperty(i)})
if assertExtra {
So(vals[i]["Extra"], ShouldResemble, []Property{MkProperty("whoa")})
}
if k.Incomplete() {
k = mkKey(k.AppID(), k.Namespace(), k.Kind(), int64(i+1), k.Parent())
}
}
cb(k, err)
}
return nil
}
func (f *fakeDatastore) DeleteMulti(keys []*Key, cb DeleteMultiCB) error {
if keys[0].Kind() == "FailAll" {
return errors.New("DeleteMulti fail all")
}
for _, k := range keys {
if k.Kind() == "Fail" {
cb(errors.New("DeleteMulti fail"))
} else {
cb(nil)
}
}
return nil
}
func (f *fakeDatastore) GetMulti(keys []Key, _meta MultiMetaGetter, cb GetMultiCB) error {
if keys[0].Kind() == "FailAll" {
return errors.New("GetMulti fail all")
}
for i, k := range keys {
if k.Kind() == "Fail" {
cb(nil, errors.New("GetMulti fail"))
} else {
cb(PropertyMap{"Value": {MkProperty(i + 1)}}, nil)
}
}
return nil
}
func (t *taskqueueImpl) deleteLocked(task *tq.Task, queueName string) error {
if _, ok := t.archived[queueName][task.Name]; ok {
return errors.New("TOMBSTONED_TASK")
}
if _, ok := t.named[queueName][task.Name]; !ok {
return errors.New("UNKNOWN_TASK")
}
t.archived[queueName][task.Name] = t.named[queueName][task.Name]
delete(t.named[queueName], task.Name)
return nil
}
func (tcf *checkFilter) PutMulti(keys []Key, vals []PropertyMap, cb PutMultiCB) error {
if len(keys) != len(vals) {
return fmt.Errorf("datastore: PutMulti with mismatched keys/vals lengths (%d/%d)", len(keys), len(vals))
}
if len(keys) == 0 {
return nil
}
if cb == nil {
return fmt.Errorf("datastore: PutMulti callback is nil")
}
lme := errors.NewLazyMultiError(len(keys))
for i, k := range keys {
if !k.PartialValid(tcf.aid, tcf.ns) {
lme.Assign(i, ErrInvalidKey)
continue
}
v := vals[i]
if v == nil {
lme.Assign(i, errors.New("datastore: PutMulti got nil vals entry"))
}
}
if me := lme.Get(); me != nil {
for _, err := range me.(errors.MultiError) {
cb(nil, err)
}
return nil
}
return tcf.RawInterface.PutMulti(keys, vals, cb)
}
func (m *memcacheImpl) Increment(key string, delta int64, initialValue *uint64) (uint64, error) {
now := clock.Now(m.ctx)
m.data.lock.Lock()
defer m.data.lock.Unlock()
cur := uint64(0)
if initialValue == nil {
curItm, err := m.data.retrieveLocked(now, key)
if err != nil {
return 0, err
}
if len(curItm.value) != 8 {
return 0, errors.New("memcache Increment: got invalid current value")
}
cur = binary.LittleEndian.Uint64(curItm.value)
} else {
cur = *initialValue
}
if delta < 0 {
if uint64(-delta) > cur {
cur = 0
} else {
cur -= uint64(-delta)
}
} else {
cur += uint64(delta)
}
newval := make([]byte, 8)
binary.LittleEndian.PutUint64(newval, cur)
m.data.setItemLocked(now, m.NewItem(key).SetValue(newval))
return cur, nil
}
// writeMutation ensures that this transaction can support the given key/value
// mutation.
//
// if getOnly is true, don't record the actual mutation data, just ensure that
// the key is in an included entity group (or add an empty entry for that
// group).
//
// if !getOnly && data == nil, this counts as a deletion instead of a Put.
//
// Returns an error if this key causes the transaction to cross too many entity
// groups.
func (td *txnDataStoreData) writeMutation(getOnly bool, key *ds.Key, data ds.PropertyMap) error {
rk := string(keyBytes(key.Root()))
td.Lock()
defer td.Unlock()
if _, ok := td.muts[rk]; !ok {
limit := 1
if td.isXG {
limit = xgEGLimit
}
if len(td.muts)+1 > limit {
msg := "cross-group transaction need to be explicitly specified (xg=True)"
if td.isXG {
msg = "operating on too many entity groups in a single transaction"
}
return errors.New(msg)
}
td.muts[rk] = []txnMutation{}
}
if !getOnly {
td.muts[rk] = append(td.muts[rk], txnMutation{key, data})
}
return nil
}
func (t *taskqueueTxnImpl) addLocked(task *tq.Task, queueName string) (*tq.Task, error) {
toSched, err := t.parent.prepTask(t.ctx, t.ns, task, queueName)
if err != nil {
return nil, err
}
numTasks := 0
for _, vs := range t.anony {
numTasks += len(vs)
}
if numTasks+1 > 5 {
// transactional tasks are actually implemented 'for real' as Actions which
// ride on the datastore. The current datastore implementation only allows
// a maximum of 5 Actions per transaction, and more than that result in a
// BAD_REQUEST.
return nil, errors.New("BAD_REQUEST")
}
t.anony[queueName] = append(t.anony[queueName], toSched)
// the fact that we have generated a unique name for this task queue item is
// an implementation detail.
// TODO(riannucci): now that I think about this... it may not actually be true.
// We should verify that the .Name for a task added in a transaction is
// meaningless. Maybe names generated in a transaction are somehow
// guaranteed to be meaningful?
toRet := toSched.Duplicate()
toRet.Name = ""
return toRet, nil
}
func (f *fakeDatastore) Run(fq *FinalizedQuery, cb RawRunCB) error {
lim, _ := fq.Limit()
cursCB := func() (Cursor, error) {
return fakeCursor("CURSOR"), nil
}
for i := int32(0); i < lim; i++ {
if v, ok := fq.eqFilts["$err_single"]; ok {
idx := fq.eqFilts["$err_single_idx"][0].Value().(int64)
if idx == int64(i) {
return errors.New(v[0].Value().(string))
}
}
k := f.mkKey("Kind", i+1)
if i == 10 {
k = f.mkKey("Kind", "eleven")
}
pm := PropertyMap{"Value": {MkProperty(i)}}
if err := cb(k, pm, cursCB); err != nil {
if err == Stop {
err = nil
}
return err
}
}
return nil
}
func newPubSubService(ctx context.Context, config pubsubConfig, client *http.Client) (pubSubService, error) {
if config.project == "" {
return nil, errors.New("pubsub: you must supply a project")
}
return &pubSubServiceImpl{
ctx: cloud.WithContext(ctx, config.project, client),
}, nil
}
func (f *FakePLS) Load(pm PropertyMap) error {
if f.failLoad {
return errors.New("FakePLS.Load")
}
f.gotLoaded = true
f.Value = pm["Value"][0].Value().(int64)
return nil
}
func (td *txnDataStoreData) run(f func() error) error {
// Slightly different from the SDK... datastore and taskqueue each implement
// this here, where in the SDK only datastore.transaction.Call does.
if atomic.LoadInt32(&td.closed) == 1 {
return errors.New("datastore: transaction context has expired")
}
return f()
}
func isOkType(t reflect.Type) error {
if t == nil {
return errors.New("no type information")
}
if t.Implements(typeOfPropertyLoadSaver) {
return nil
}
if t == typeOfKey {
return errors.New("not user datatype")
}
if t.Kind() != reflect.Ptr {
return errors.New("not a pointer")
}
if t.Elem().Kind() != reflect.Struct {
return errors.New("does not point to a struct")
}
return nil
}
func (d *datastoreImpl) GetAll(q *Query, dst interface{}) error {
v := reflect.ValueOf(dst)
if v.Kind() != reflect.Ptr {
panic(fmt.Errorf("invalid GetAll dst: must have a ptr-to-slice: %T", dst))
}
if !v.IsValid() || v.IsNil() {
panic(errors.New("invalid GetAll dst: <nil>"))
}
if keys, ok := dst.(*[]*Key); ok {
fq, err := q.KeysOnly(true).Finalize()
if err != nil {
return err
}
return d.RawInterface.Run(fq, func(k *Key, _ PropertyMap, _ CursorCB) error {
*keys = append(*keys, k)
return nil
})
}
fq, err := q.Finalize()
if err != nil {
return err
}
slice := v.Elem()
mat := parseMultiArg(slice.Type())
if mat.newElem == nil {
panic(fmt.Errorf("invalid GetAll dst (non-concrete element type): %T", dst))
}
errs := map[int]error{}
i := 0
err = d.RawInterface.Run(fq, func(k *Key, pm PropertyMap, _ CursorCB) error {
slice.Set(reflect.Append(slice, mat.newElem()))
itm := slice.Index(i)
mat.setKey(itm, k)
err := mat.setPM(itm, pm)
if err != nil {
errs[i] = err
}
i++
return nil
})
if err == nil {
if len(errs) > 0 {
me := make(errors.MultiError, slice.Len())
for i, e := range errs {
me[i] = e
}
err = me
}
}
return err
}
func (s *testPubSubService) Pull(sub string, count int) (msgs []*pubsub.Message, err error) {
mock, e := s.PopErr("Pull", sub, count)
if e != nil {
if s.infinitePull {
return nil, nil
}
s.AddError(errors.New("out of mock Pull entries"))
}
mock.BindResult(&msgs, &err)
return
}
func (d *dataStoreData) allocateIDsLocked(ents *memCollection, incomplete *ds.Key, n int) (int64, error) {
if d.disableSpecialEntities {
return 0, errors.New("disableSpecialEntities is true so allocateIDs is disabled")
}
idKey := []byte(nil)
if incomplete.Parent() == nil {
idKey = rootIDsKey(incomplete.Kind())
} else {
idKey = groupIDsKey(incomplete)
}
return incrementLocked(ents, idKey, n), nil
}
func TestBrokenFeatures(t *testing.T) {
t.Parallel()
e := errors.New("default err")
Convey("BrokenFeatures", t, func() {
c := memory.Use(context.Background())
Convey("Can break ds", func() {
Convey("without a default", func() {
c, bf := FilterRDS(c, nil)
ds := datastore.Get(c)
vals := []datastore.PropertyMap{{
"$key": {datastore.MkPropertyNI(ds.NewKey("Wut", "", 1, nil))},
}}
Convey("by specifying an error", func() {
bf.BreakFeatures(e, "GetMulti", "PutMulti")
So(ds.GetMulti(vals), ShouldEqual, e)
Convey("and you can unbreak them as well", func() {
bf.UnbreakFeatures("GetMulti")
So(errors.SingleError(ds.GetMulti(vals)), ShouldEqual, datastore.ErrNoSuchEntity)
Convey("no broken features at all is a shortcut", func() {
bf.UnbreakFeatures("PutMulti")
So(errors.SingleError(ds.GetMulti(vals)), ShouldEqual, datastore.ErrNoSuchEntity)
})
})
})
Convey("Not specifying an error gets you a generic error", func() {
bf.BreakFeatures(nil, "GetMulti")
So(ds.GetMulti(vals).Error(), ShouldContainSubstring, `feature "GetMulti" is broken`)
})
})
Convey("with a default", func() {
c, bf := FilterRDS(c, e)
ds := datastore.Get(c)
vals := []datastore.PropertyMap{{
"$key": {datastore.MkPropertyNI(ds.NewKey("Wut", "", 1, nil))},
}}
bf.BreakFeatures(nil, "GetMulti")
So(ds.GetMulti(vals), ShouldEqual, e)
})
})
})
}
func (c *endpointConfig) createService(ctx context.Context) (endpointService, error) {
if c.url == "" {
return nil, errors.New("endpoint: you must supply a monitoring endpoint")
}
authenticator := auth.NewAuthenticator(
auth.Options{
Method: auth.ServiceAccountMethod,
Scopes: endpointScopes,
ServiceAccountJSONPath: c.serviceAccountJSONPath,
Logger: log.Get(ctx),
})
client, err := auth.AuthenticatedClient(auth.SilentLogin, authenticator)
if err != nil {
log.Errorf(log.SetError(ctx, err), "Failed to configure endpoint client.")
return nil, errors.New("endpoint: failed to configure endpoint client")
}
return &endpointServiceImpl{
endpointConfig: *c,
client: client,
}, nil
}
// newPubSubClient instantiates a new Pub/Sub client.
//
// This method will also perform authentication and setup the topic/subscription
// if it isn't already set up.
func newPubSubClient(ctx context.Context, config pubsubConfig, svc pubSubService) (*pubsubClient, error) {
if config.subscription == "" {
return nil, errors.New("pubsub: you must supply a subscription")
}
if config.batchSize <= 0 {
return nil, errors.New("pubsub: batch size must be at least 1")
} else if config.batchSize > maxSubscriptionPullSize {
return nil, fmt.Errorf("pubsub: batch size cannot exceed %d", maxSubscriptionPullSize)
}
p := pubsubClient{
pubsubConfig: &config,
service: svc,
}
// Ensure that our Subscription (and topic) exist.
if err := p.setupSubscription(ctx); err != nil {
log.Errorf(log.SetError(ctx, err), "Failed to set up subscription.")
return nil, err
}
return &p, nil
}
func (f *FakePLS) GetMeta(key string) (interface{}, error) {
if f.failGetMeta {
return nil, errors.New("FakePLS.GetMeta")
}
switch key {
case "id":
if f.StringID != "" {
return f.StringID, nil
}
return f.IntID, nil
case "kind":
if f.Kind == "" {
return "FakePLS", nil
}
return f.Kind, nil
}
return nil, ErrMetaFieldUnset
}
func (t *taskqueueTxnImpl) AddMulti(tasks []*tq.Task, queueName string, cb tq.RawTaskCB) error {
if atomic.LoadInt32(&t.closed) == 1 {
return errors.New("taskqueue: transaction context has expired")
}
t.Lock()
defer t.Unlock()
queueName, err := t.parent.getQueueNameLocked(queueName)
if err != nil {
return err
}
for _, task := range tasks {
cb(t.addLocked(task, queueName))
}
return nil
}
func (f *FakePLS) SetMeta(key string, val interface{}) error {
if f.failSetMeta {
return errors.New("FakePL.SetMeta")
}
if key == "id" {
switch x := val.(type) {
case int64:
f.IntID = x
case string:
f.StringID = x
}
return nil
}
if key == "kind" {
f.Kind = val.(string)
return nil
}
return ErrMetaFieldUnset
}
func (d *dsTxnBuf) Run(fq *ds.FinalizedQuery, cb ds.RawRunCB) error {
if start, end := fq.Bounds(); start != nil || end != nil {
return errors.New("txnBuf filter does not support query cursors")
}
limit, limitSet := fq.Limit()
offset, _ := fq.Offset()
keysOnly := fq.KeysOnly()
project := fq.Project()
bufDS, parentDS, sizes := func() (ds.RawInterface, ds.RawInterface, *sizeTracker) {
if !d.haveLock {
d.state.Lock()
defer d.state.Unlock()
}
return d.state.bufDS, d.state.parentDS, d.state.entState.dup()
}()
return runMergedQueries(fq, sizes, bufDS, parentDS, func(key *ds.Key, data ds.PropertyMap) error {
if offset > 0 {
offset--
return nil
}
if limitSet {
if limit == 0 {
return ds.Stop
}
limit--
}
if keysOnly {
data = nil
} else if len(project) > 0 {
newData := make(ds.PropertyMap, len(project))
for _, p := range project {
newData[p] = data[p]
}
data = newData
}
return cb(key, data, nil)
})
}
func (f *FakePLS) Save(withMeta bool) (PropertyMap, error) {
if f.failSave {
return nil, errors.New("FakePLS.Save")
}
ret := PropertyMap{
"Value": {MkProperty(f.Value)},
"Extra": {MkProperty("whoa")},
}
if withMeta {
id, _ := f.GetMeta("id")
So(ret.SetMeta("id", id), ShouldBeTrue)
if f.Kind == "" {
So(ret.SetMeta("kind", "FakePLS"), ShouldBeTrue)
} else {
So(ret.SetMeta("kind", f.Kind), ShouldBeTrue)
}
So(ret.SetMeta("assertExtra", true), ShouldBeTrue)
}
return ret, nil
}
func (d *datastoreImpl) GetAll(q Query, dst interface{}) error {
v := reflect.ValueOf(dst)
if v.Kind() != reflect.Ptr {
return fmt.Errorf("invalid GetAll dst: must have a ptr-to-slice: %T", dst)
}
if !v.IsValid() || v.IsNil() {
return errors.New("invalid GetAll dst: <nil>")
}
if keys, ok := dst.(*[]Key); ok {
return d.RawInterface.Run(q.KeysOnly(), func(k Key, _ PropertyMap, _ CursorCB) bool {
*keys = append(*keys, k)
return true
})
}
slice := v.Elem()
mat := parseMultiArg(slice.Type())
if !mat.valid || mat.newElem == nil {
return fmt.Errorf("invalid GetAll input type: %T", dst)
}
lme := errors.NewLazyMultiError(slice.Len())
i := 0
err := d.RawInterface.Run(q, func(k Key, pm PropertyMap, _ CursorCB) bool {
slice.Set(reflect.Append(slice, mat.newElem()))
itm := slice.Index(i)
mat.setKey(itm, k)
lme.Assign(i, mat.setPM(itm, pm))
i++
return true
})
if err == nil {
err = lme.Get()
}
return err
}
func (p *structPLS) save(propMap PropertyMap, prefix string, is IndexSetting) (idxCount int, err error) {
saveProp := func(name string, si IndexSetting, v reflect.Value, st *structTag) (err error) {
if st.substructCodec != nil {
count, err := (&structPLS{v, st.substructCodec}).save(propMap, name, si)
if err == nil {
idxCount += count
if idxCount > maxIndexedProperties {
err = errors.New("gae: too many indexed properties")
}
}
return err
}
prop := Property{}
if st.convert {
prop, err = v.Addr().Interface().(PropertyConverter).ToProperty()
} else {
err = prop.SetValue(v.Interface(), si)
}
if err != nil {
return err
}
propMap[name] = append(propMap[name], prop)
if prop.IndexSetting() == ShouldIndex {
idxCount++
if idxCount > maxIndexedProperties {
return errors.New("gae: too many indexed properties")
}
}
return nil
}
for i, st := range p.c.byIndex {
if st.name == "-" || st.isExtra {
continue
}
name := st.name
if prefix != "" {
name = prefix + name
}
v := p.o.Field(i)
is1 := is
if st.idxSetting == NoIndex {
is1 = NoIndex
}
if st.isSlice {
for j := 0; j < v.Len(); j++ {
if err = saveProp(name, is1, v.Index(j), &st); err != nil {
return
}
}
} else {
if err = saveProp(name, is1, v, &st); err != nil {
return
}
}
}
if i, ok := p.c.bySpecial["extra"]; ok {
if p.c.byIndex[i].name != "-" {
for fullName, vals := range p.o.Field(i).Interface().(PropertyMap) {
if _, ok := propMap[fullName]; !ok {
propMap[fullName] = vals
}
}
}
}
return
}
func (t *taskqueueTxnImpl) Stats([]string, tq.RawStatsCB) error {
return errors.New("taskqueue: cannot Stats from a transaction")
}
)
const (
// The maximum number of items that can be pulled from a subscription at once.
maxSubscriptionPullSize = 100
)
var (
// OAuth2 scopes to generate.
pubsubScopes = []string{
pubsub.ScopePubSub,
auth.OAuthScopeEmail,
}
// Error returned by pullAckMessages to indicate that no messages were available.
errNoMessages = errors.New("pubsub: no messages")
)
// pubsubConfig is the set of configuration parameters for a pubsubClient.
type pubsubConfig struct {
project string // The project name.
topic string // The topic name.
subscription string // The subscription name.
create bool
batchSize int // The number of elements to pull from a subscription per batch.
}
// addFlags adds this configuration's set of flags to a FlagSet.
func (c *pubsubConfig) addFlags(fs *flag.FlagSet) {
fs.StringVar(&c.project, "pubsub-project", "", "The name of the Pub/Sub project.")
fs.StringVar(&c.subscription, "pubsub-subscription", "", "The name of the Pub/Sub subscription.")
// setupSubscription asserts that the configured subscription exists. In doing
// so, it also asserts that the client credentials are valid with respect to the
// configured project/subscription.
//
// If the subscription doesn't exist, this method can create the subscription
// and (if missing) its topic, if the "create" flag is set.
func (p *pubsubClient) setupSubscription(ctx context.Context) error {
exists := false
log.Fields{
"subscription": p.topic,
}.Infof(ctx, "Checking for subscription existence.")
err := retryCall(ctx, "SubExists()", func() error {
var err error
exists, err = p.service.SubExists(p.subscription)
return p.wrapTransient(err)
})
if err != nil {
log.Warningf(log.SetError(ctx, err),
"Failed to test for subscription; assuming it doesn't exist.")
}
if exists {
return nil
}
if !p.create {
return errors.New("pubsub: subscription doesn't exist, not configured to create")
}
// Create the subscription if it doesn't exist.
if p.topic == "" {
log.Errorf(ctx, "Cannot create subscription; no topic was specified.")
return errors.New("pubsub: cannot create subscription")
}
// Test if the topic exists...
log.Fields{
"topic": p.topic,
}.Infof(ctx, "Checking for topic existence.")
err = retryCall(ctx, "TopicExists()", func() error {
var err error
exists, err = p.service.TopicExists(p.topic)
return p.wrapTransient(err)
})
if err != nil {
log.Warningf(log.SetError(ctx, err),
"Failed to test for topic; assuming it doesn't exist.")
}
if !exists {
log.Fields{
"topic": p.topic,
}.Infof(ctx, "Creating topic.")
err := retryCall(ctx, "CreateTopic()", func() error {
return p.service.CreateTopic(p.topic)
})
if err != nil {
log.Warningf(log.SetError(ctx, err),
"Failed to create topic.")
return errors.New("pubsub: cannot create topic")
}
}
log.Fields{
"topic": p.topic,
"subscription": p.subscription,
}.Infof(ctx, "Creating pull subscription for topic.")
if err := retryCall(ctx, "CreateSub()", func() error {
return p.service.CreatePullSub(p.subscription, p.topic)
}); err != nil {
log.Warningf(log.SetError(ctx, err),
"Failed to test for subscription; assuming it doesn't exist.")
return errors.New("pubsub: failed to create subscription")
}
return nil
}