// DoPerRPCCreds performs a unary RPC with per RPC OAUTH2 token. func DoPerRPCCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) { jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile) pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(), ResponseSize: proto.Int32(int32(largeRespSize)), Payload: pl, FillUsername: proto.Bool(true), FillOauthScope: proto.Bool(true), } token := GetToken(serviceAccountKeyFile, oauthScope) kv := map[string]string{"authorization": token.TokenType + " " + token.AccessToken} ctx := metadata.NewContext(context.Background(), metadata.MD{"authorization": []string{kv["authorization"]}}) reply, err := tc.UnaryCall(ctx, req) if err != nil { grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err) } user := reply.GetUsername() scope := reply.GetOauthScope() if !strings.Contains(string(jsonKey), user) { grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey) } if !strings.Contains(oauthScope, scope) { grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope) } grpclog.Println("PerRPCCreds done") }
func sortToProto(sort *SortOptions, params *pb.SearchParams) error { for _, e := range sort.Expressions { spec := &pb.SortSpec{ SortExpression: proto.String(e.Expr), } if e.Reverse { spec.SortDescending = proto.Bool(false) } if e.Default != nil { switch d := e.Default.(type) { case float64: spec.DefaultValueNumeric = &d case string: spec.DefaultValueText = &d default: return fmt.Errorf("search: invalid Default type %T for expression %q", d, e.Expr) } } params.SortSpec = append(params.SortSpec, spec) } spec := &pb.ScorerSpec{} if sort.Limit > 0 { spec.Limit = proto.Int32(int32(sort.Limit)) params.ScorerSpec = spec } if sort.Scorer != nil { sort.Scorer.toProto(spec) params.ScorerSpec = spec } return nil }
func lease(c context.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) { if queueName == "" { queueName = "default" } req := &pb.TaskQueueQueryAndOwnTasksRequest{ QueueName: []byte(queueName), LeaseSeconds: proto.Float64(float64(leaseTime)), MaxTasks: proto.Int64(int64(maxTasks)), GroupByTag: proto.Bool(groupByTag), Tag: tag, } res := &pb.TaskQueueQueryAndOwnTasksResponse{} if err := internal.Call(c, "taskqueue", "QueryAndOwnTasks", req, res); err != nil { return nil, err } tasks := make([]*Task, len(res.Task)) for i, t := range res.Task { tasks[i] = &Task{ Payload: t.Body, Name: string(t.TaskName), Method: "PULL", ETA: time.Unix(0, *t.EtaUsec*1e3), RetryCount: *t.RetryCount, Tag: string(t.Tag), } } return tasks, nil }
func RunTransactionOnce(c netcontext.Context, f func(netcontext.Context) error, xg bool) error { if transactionFromContext(c) != nil { return errors.New("nested transactions are not supported") } // Begin the transaction. t := &transaction{} req := &pb.BeginTransactionRequest{ App: proto.String(FullyQualifiedAppID(c)), } if xg { req.AllowMultipleEg = proto.Bool(true) } if err := Call(c, "datastore_v3", "BeginTransaction", req, &t.transaction); err != nil { return err } // Call f, rolling back the transaction if f returns a non-nil error, or panics. // The panic is not recovered. defer func() { if t.finished { return } t.finished = true // Ignore the error return value, since we are already returning a non-nil // error (or we're panicking). Call(c, "datastore_v3", "Rollback", &t.transaction, &basepb.VoidProto{}) }() if err := f(withTransaction(c, t)); err != nil { return err } t.finished = true // Commit the transaction. res := &pb.CommitResponse{} err := Call(c, "datastore_v3", "Commit", &t.transaction, res) if ae, ok := err.(*APIError); ok { /* TODO: restore this conditional if appengine.IsDevAppServer() { */ // The Python Dev AppServer raises an ApplicationError with error code 2 (which is // Error.CONCURRENT_TRANSACTION) and message "Concurrency exception.". if ae.Code == int32(pb.Error_BAD_REQUEST) && ae.Detail == "ApplicationError: 2 Concurrency exception." { return ErrConcurrentTransaction } if ae.Code == int32(pb.Error_CONCURRENT_TRANSACTION) { return ErrConcurrentTransaction } } return err }
func propertiesToProto(key *Key, props []Property) (*pb.Entity, error) { e := &pb.Entity{ Key: keyToProto(key), } indexedProps := 0 prevMultiple := make(map[string]*pb.Property) for _, p := range props { val, err := interfaceToProto(p.Value) if err != "" { return nil, fmt.Errorf("datastore: %s for a Property with Name %q", err, p.Name) } if !p.NoIndex { rVal := reflect.ValueOf(p.Value) if rVal.Kind() == reflect.Slice && rVal.Type().Elem().Kind() != reflect.Uint8 { indexedProps += rVal.Len() } else { indexedProps++ } } if indexedProps > maxIndexedProperties { return nil, errors.New("datastore: too many indexed properties") } switch v := p.Value.(type) { case string: case []byte: if len(v) > 1500 && !p.NoIndex { return nil, fmt.Errorf("datastore: cannot index a Property with Name %q", p.Name) } } val.Indexed = proto.Bool(!p.NoIndex) if p.Multiple { x, ok := prevMultiple[p.Name] if !ok { x = &pb.Property{ Name: proto.String(p.Name), Value: &pb.Value{}, } prevMultiple[p.Name] = x e.Property = append(e.Property, x) } x.Value.ListValue = append(x.Value.ListValue, val) } else { e.Property = append(e.Property, &pb.Property{ Name: proto.String(p.Name), Value: val, }) } } return e, nil }
// DoComputeEngineCreds performs a unary RPC with compute engine auth. func DoComputeEngineCreds(tc testpb.TestServiceClient, serviceAccount, oauthScope string) { pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(), ResponseSize: proto.Int32(int32(largeRespSize)), Payload: pl, FillUsername: proto.Bool(true), FillOauthScope: proto.Bool(true), } reply, err := tc.UnaryCall(context.Background(), req) if err != nil { grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err) } user := reply.GetUsername() scope := reply.GetOauthScope() if user != serviceAccount { grpclog.Fatalf("Got user name %q, want %q.", user, serviceAccount) } if !strings.Contains(oauthScope, scope) { grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope) } grpclog.Println("ComputeEngineCreds done") }
// callNext issues a datastore_v3/Next RPC to advance a cursor, such as that // returned by a query with more results. func callNext(c context.Context, res *pb.QueryResult, offset, limit int32) error { if res.Cursor == nil { return errors.New("datastore: internal error: server did not return a cursor") } req := &pb.NextRequest{ Cursor: res.Cursor, } if limit >= 0 { req.Count = proto.Int32(limit) } if offset != 0 { req.Offset = proto.Int32(offset) } if res.CompiledCursor != nil { req.Compile = proto.Bool(true) } res.Reset() return internal.Call(c, "datastore_v3", "Next", req, res) }
// DoJWTTokenCreds performs a unary RPC with JWT token auth. func DoJWTTokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile string) { pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(), ResponseSize: proto.Int32(int32(largeRespSize)), Payload: pl, FillUsername: proto.Bool(true), } reply, err := tc.UnaryCall(context.Background(), req) if err != nil { grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err) } jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile) user := reply.GetUsername() if !strings.Contains(string(jsonKey), user) { grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey) } grpclog.Println("JWTtokenCreds done") }
// GetMulti is a batch version of Get. The returned map from keys to items may // have fewer elements than the input slice, due to memcache cache misses. // Each key must be at most 250 bytes in length. func GetMulti(c context.Context, key []string) (map[string]*Item, error) { if len(key) == 0 { return nil, nil } keyAsBytes := make([][]byte, len(key)) for i, k := range key { keyAsBytes[i] = []byte(k) } req := &pb.MemcacheGetRequest{ Key: keyAsBytes, ForCas: proto.Bool(true), } res := &pb.MemcacheGetResponse{} if err := internal.Call(c, "memcache", "Get", req, res); err != nil { return nil, err } m := make(map[string]*Item, len(res.Item)) for _, p := range res.Item { t := protoToItem(p) m[t.Key] = t } return m, nil }
func interfaceToProto(iv interface{}) (p *pb.Value, errStr string) { val := new(pb.Value) switch v := iv.(type) { case int: val.IntegerValue = proto.Int64(int64(v)) case int32: val.IntegerValue = proto.Int64(int64(v)) case int64: val.IntegerValue = proto.Int64(v) case bool: val.BooleanValue = proto.Bool(v) case string: val.StringValue = proto.String(v) case float32: val.DoubleValue = proto.Float64(float64(v)) case float64: val.DoubleValue = proto.Float64(v) case *Key: if v != nil { val.KeyValue = keyToProto(v) } case time.Time: if v.Before(minTime) || v.After(maxTime) { return nil, fmt.Sprintf("time value out of range") } val.TimestampMicrosecondsValue = proto.Int64(toUnixMicro(v)) case []byte: val.BlobValue = v default: if iv != nil { return nil, fmt.Sprintf("invalid Value type %t", iv) } } // TODO(jbd): Support ListValue and EntityValue. // TODO(jbd): Support types whose underlying type is one of the types above. return val, "" }
// toProto converts the query to a protocol buffer. func (q *Query) toProto(dst *pb.Query, appID string) error { if len(q.projection) != 0 && q.keysOnly { return errors.New("datastore: query cannot both project and be keys-only") } dst.Reset() dst.App = proto.String(appID) if q.kind != "" { dst.Kind = proto.String(q.kind) } if q.ancestor != nil { dst.Ancestor = keyToProto(appID, q.ancestor) if q.eventual { dst.Strong = proto.Bool(false) } } if q.projection != nil { dst.PropertyName = q.projection if q.distinct { dst.GroupByPropertyName = q.projection } } if q.keysOnly { dst.KeysOnly = proto.Bool(true) dst.RequirePerfectPlan = proto.Bool(true) } for _, qf := range q.filter { if qf.FieldName == "" { return errors.New("datastore: empty query filter field name") } p, errStr := valueToProto(appID, qf.FieldName, reflect.ValueOf(qf.Value), false) if errStr != "" { return errors.New("datastore: bad query filter value type: " + errStr) } xf := &pb.Query_Filter{ Op: operatorToProto[qf.Op], Property: []*pb.Property{p}, } if xf.Op == nil { return errors.New("datastore: unknown query filter operator") } dst.Filter = append(dst.Filter, xf) } for _, qo := range q.order { if qo.FieldName == "" { return errors.New("datastore: empty query order field name") } xo := &pb.Query_Order{ Property: proto.String(qo.FieldName), Direction: sortDirectionToProto[qo.Direction], } if xo.Direction == nil { return errors.New("datastore: unknown query order direction") } dst.Order = append(dst.Order, xo) } if q.limit >= 0 { dst.Limit = proto.Int32(q.limit) } if q.offset != 0 { dst.Offset = proto.Int32(q.offset) } dst.CompiledCursor = q.start dst.EndCompiledCursor = q.end dst.Compile = proto.Bool(true) return nil }
// valueToProto converts a named value to a newly allocated Property. // The returned error string is empty on success. func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) { var ( pv pb.PropertyValue unsupported bool ) switch v.Kind() { case reflect.Invalid: // No-op. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: pv.Int64Value = proto.Int64(v.Int()) case reflect.Bool: pv.BooleanValue = proto.Bool(v.Bool()) case reflect.String: pv.StringValue = proto.String(v.String()) case reflect.Float32, reflect.Float64: pv.DoubleValue = proto.Float64(v.Float()) case reflect.Ptr: if k, ok := v.Interface().(*Key); ok { if k != nil { pv.Referencevalue = keyToReferenceValue(defaultAppID, k) } } else { unsupported = true } case reflect.Struct: switch t := v.Interface().(type) { case time.Time: if t.Before(minTime) || t.After(maxTime) { return nil, "time value out of range" } pv.Int64Value = proto.Int64(toUnixMicro(t)) case appengine.GeoPoint: if !t.Valid() { return nil, "invalid GeoPoint value" } // NOTE: Strangely, latitude maps to X, longitude to Y. pv.Pointvalue = &pb.PropertyValue_PointValue{X: &t.Lat, Y: &t.Lng} default: unsupported = true } case reflect.Slice: if b, ok := v.Interface().([]byte); ok { pv.StringValue = proto.String(string(b)) } else { // nvToProto should already catch slice values. // If we get here, we have a slice of slice values. unsupported = true } default: unsupported = true } if unsupported { return nil, "unsupported datastore value type: " + v.Type().String() } p = &pb.Property{ Name: proto.String(name), Value: &pv, Multiple: proto.Bool(multiple), } if v.IsValid() { switch v.Interface().(type) { case []byte: p.Meaning = pb.Property_BLOB.Enum() case ByteString: p.Meaning = pb.Property_BYTESTRING.Enum() case appengine.BlobKey: p.Meaning = pb.Property_BLOBKEY.Enum() case time.Time: p.Meaning = pb.Property_GD_WHEN.Enum() case appengine.GeoPoint: p.Meaning = pb.Property_GEORSS_POINT.Enum() } } return p, "" }
func propertiesToProto(defaultAppID string, key *Key, props []Property) (*pb.EntityProto, error) { e := &pb.EntityProto{ Key: keyToProto(defaultAppID, key), } if key.parent == nil { e.EntityGroup = &pb.Path{} } else { e.EntityGroup = keyToProto(defaultAppID, key.root()).Path } prevMultiple := make(map[string]bool) for _, p := range props { if pm, ok := prevMultiple[p.Name]; ok { if !pm || !p.Multiple { return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name) } } else { prevMultiple[p.Name] = p.Multiple } x := &pb.Property{ Name: proto.String(p.Name), Value: new(pb.PropertyValue), Multiple: proto.Bool(p.Multiple), } switch v := p.Value.(type) { case int64: x.Value.Int64Value = proto.Int64(v) case bool: x.Value.BooleanValue = proto.Bool(v) case string: x.Value.StringValue = proto.String(v) if p.NoIndex { x.Meaning = pb.Property_TEXT.Enum() } case float64: x.Value.DoubleValue = proto.Float64(v) case *Key: if v != nil { x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v) } case time.Time: if v.Before(minTime) || v.After(maxTime) { return nil, fmt.Errorf("datastore: time value out of range") } x.Value.Int64Value = proto.Int64(toUnixMicro(v)) x.Meaning = pb.Property_GD_WHEN.Enum() case appengine.BlobKey: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BLOBKEY.Enum() case appengine.GeoPoint: if !v.Valid() { return nil, fmt.Errorf("datastore: invalid GeoPoint value") } // NOTE: Strangely, latitude maps to X, longitude to Y. x.Value.Pointvalue = &pb.PropertyValue_PointValue{X: &v.Lat, Y: &v.Lng} x.Meaning = pb.Property_GEORSS_POINT.Enum() case []byte: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BLOB.Enum() if !p.NoIndex { return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name) } case ByteString: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BYTESTRING.Enum() default: if p.Value != nil { return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name) } } if p.NoIndex { e.RawProperty = append(e.RawProperty, x) } else { e.Property = append(e.Property, x) if len(e.Property) > maxIndexedProperties { return nil, errors.New("datastore: too many indexed properties") } } } return e, nil }
// RoundTrip issues a single HTTP request and returns its response. Per the // http.RoundTripper interface, RoundTrip only returns an error if there // was an unsupported request or the URL Fetch proxy fails. // Note that HTTP response codes such as 5xx, 403, 404, etc are not // errors as far as the transport is concerned and will be returned // with err set to nil. func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) { methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method] if !ok { return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method) } method := pb.URLFetchRequest_RequestMethod(methNum) freq := &pb.URLFetchRequest{ Method: &method, Url: proto.String(urlString(req.URL)), FollowRedirects: proto.Bool(false), // http.Client's responsibility MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate), } if deadline, ok := t.Context.Deadline(); ok { freq.Deadline = proto.Float64(deadline.Sub(time.Now()).Seconds()) } for k, vals := range req.Header { for _, val := range vals { freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{ Key: proto.String(k), Value: proto.String(val), }) } } if methodAcceptsRequestBody[req.Method] && req.Body != nil { // Avoid a []byte copy if req.Body has a Bytes method. switch b := req.Body.(type) { case interface { Bytes() []byte }: freq.Payload = b.Bytes() default: freq.Payload, err = ioutil.ReadAll(req.Body) if err != nil { return nil, err } } } fres := &pb.URLFetchResponse{} if err := internal.Call(t.Context, "urlfetch", "Fetch", freq, fres); err != nil { return nil, err } res = &http.Response{} res.StatusCode = int(*fres.StatusCode) res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode)) res.Header = make(http.Header) res.Request = req // Faked: res.ProtoMajor = 1 res.ProtoMinor = 1 res.Proto = "HTTP/1.1" res.Close = true for _, h := range fres.Header { hkey := http.CanonicalHeaderKey(*h.Key) hval := *h.Value if hkey == "Content-Length" { // Will get filled in below for all but HEAD requests. if req.Method == "HEAD" { res.ContentLength, _ = strconv.ParseInt(hval, 10, 64) } continue } res.Header.Add(hkey, hval) } if req.Method != "HEAD" { res.ContentLength = int64(len(fres.Content)) } truncated := fres.GetContentWasTruncated() res.Body = &bodyReader{content: fres.Content, truncated: truncated} return }
// set sets the given items using the given conflict resolution policy. // appengine.MultiError may be returned. func set(c context.Context, item []*Item, value [][]byte, policy pb.MemcacheSetRequest_SetPolicy) error { if len(item) == 0 { return nil } req := &pb.MemcacheSetRequest{ Item: make([]*pb.MemcacheSetRequest_Item, len(item)), } for i, t := range item { p := &pb.MemcacheSetRequest_Item{ Key: []byte(t.Key), } if value == nil { p.Value = t.Value } else { p.Value = value[i] } if t.Flags != 0 { p.Flags = proto.Uint32(t.Flags) } if t.Expiration != 0 { // In the .proto file, MemcacheSetRequest_Item uses a fixed32 (i.e. unsigned) // for expiration time, while MemcacheGetRequest_Item uses int32 (i.e. signed). // Throughout this .go file, we use int32. // Also, in the proto, the expiration value is either a duration (in seconds) // or an absolute Unix timestamp (in seconds), depending on whether the // value is less than or greater than or equal to 30 years, respectively. if t.Expiration < time.Second { // Because an Expiration of 0 means no expiration, we take // care here to translate an item with an expiration // Duration between 0-1 seconds as immediately expiring // (saying it expired a few seconds ago), rather than // rounding it down to 0 and making it live forever. p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) - 5) } else if t.Expiration >= thirtyYears { p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) + uint32(t.Expiration/time.Second)) } else { p.ExpirationTime = proto.Uint32(uint32(t.Expiration / time.Second)) } } if t.casID != 0 { p.CasId = proto.Uint64(t.casID) p.ForCas = proto.Bool(true) } p.SetPolicy = policy.Enum() req.Item[i] = p } res := &pb.MemcacheSetResponse{} if err := internal.Call(c, "memcache", "Set", req, res); err != nil { return err } if len(res.SetStatus) != len(item) { return ErrServerError } me, any := make(appengine.MultiError, len(item)), false for i, st := range res.SetStatus { var err error switch st { case pb.MemcacheSetResponse_STORED: // OK case pb.MemcacheSetResponse_NOT_STORED: err = ErrNotStored case pb.MemcacheSetResponse_EXISTS: err = ErrCASConflict default: err = ErrServerError } if err != nil { me[i] = err any = true } } if any { return me } return nil }