// DoPerRPCCreds performs a unary RPC with per RPC OAUTH2 token.
func DoPerRPCCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) {
jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
req := &testpb.SimpleRequest{
ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(),
ResponseSize: proto.Int32(int32(largeRespSize)),
Payload: pl,
FillUsername: proto.Bool(true),
FillOauthScope: proto.Bool(true),
}
token := GetToken(serviceAccountKeyFile, oauthScope)
kv := map[string]string{"authorization": token.TokenType + " " + token.AccessToken}
ctx := metadata.NewContext(context.Background(), metadata.MD{"authorization": []string{kv["authorization"]}})
reply, err := tc.UnaryCall(ctx, req)
if err != nil {
grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
}
user := reply.GetUsername()
scope := reply.GetOauthScope()
if !strings.Contains(string(jsonKey), user) {
grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
}
if !strings.Contains(oauthScope, scope) {
grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
}
grpclog.Println("PerRPCCreds done")
}
func sortToProto(sort *SortOptions, params *pb.SearchParams) error {
for _, e := range sort.Expressions {
spec := &pb.SortSpec{
SortExpression: proto.String(e.Expr),
}
if e.Reverse {
spec.SortDescending = proto.Bool(false)
}
if e.Default != nil {
switch d := e.Default.(type) {
case float64:
spec.DefaultValueNumeric = &d
case string:
spec.DefaultValueText = &d
default:
return fmt.Errorf("search: invalid Default type %T for expression %q", d, e.Expr)
}
}
params.SortSpec = append(params.SortSpec, spec)
}
spec := &pb.ScorerSpec{}
if sort.Limit > 0 {
spec.Limit = proto.Int32(int32(sort.Limit))
params.ScorerSpec = spec
}
if sort.Scorer != nil {
sort.Scorer.toProto(spec)
params.ScorerSpec = spec
}
return nil
}
func lease(c context.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) {
if queueName == "" {
queueName = "default"
}
req := &pb.TaskQueueQueryAndOwnTasksRequest{
QueueName: []byte(queueName),
LeaseSeconds: proto.Float64(float64(leaseTime)),
MaxTasks: proto.Int64(int64(maxTasks)),
GroupByTag: proto.Bool(groupByTag),
Tag: tag,
}
res := &pb.TaskQueueQueryAndOwnTasksResponse{}
if err := internal.Call(c, "taskqueue", "QueryAndOwnTasks", req, res); err != nil {
return nil, err
}
tasks := make([]*Task, len(res.Task))
for i, t := range res.Task {
tasks[i] = &Task{
Payload: t.Body,
Name: string(t.TaskName),
Method: "PULL",
ETA: time.Unix(0, *t.EtaUsec*1e3),
RetryCount: *t.RetryCount,
Tag: string(t.Tag),
}
}
return tasks, nil
}
func RunTransactionOnce(c netcontext.Context, f func(netcontext.Context) error, xg bool) error {
if transactionFromContext(c) != nil {
return errors.New("nested transactions are not supported")
}
// Begin the transaction.
t := &transaction{}
req := &pb.BeginTransactionRequest{
App: proto.String(FullyQualifiedAppID(c)),
}
if xg {
req.AllowMultipleEg = proto.Bool(true)
}
if err := Call(c, "datastore_v3", "BeginTransaction", req, &t.transaction); err != nil {
return err
}
// Call f, rolling back the transaction if f returns a non-nil error, or panics.
// The panic is not recovered.
defer func() {
if t.finished {
return
}
t.finished = true
// Ignore the error return value, since we are already returning a non-nil
// error (or we're panicking).
Call(c, "datastore_v3", "Rollback", &t.transaction, &basepb.VoidProto{})
}()
if err := f(withTransaction(c, t)); err != nil {
return err
}
t.finished = true
// Commit the transaction.
res := &pb.CommitResponse{}
err := Call(c, "datastore_v3", "Commit", &t.transaction, res)
if ae, ok := err.(*APIError); ok {
/* TODO: restore this conditional
if appengine.IsDevAppServer() {
*/
// The Python Dev AppServer raises an ApplicationError with error code 2 (which is
// Error.CONCURRENT_TRANSACTION) and message "Concurrency exception.".
if ae.Code == int32(pb.Error_BAD_REQUEST) && ae.Detail == "ApplicationError: 2 Concurrency exception." {
return ErrConcurrentTransaction
}
if ae.Code == int32(pb.Error_CONCURRENT_TRANSACTION) {
return ErrConcurrentTransaction
}
}
return err
}
func propertiesToProto(key *Key, props []Property) (*pb.Entity, error) {
e := &pb.Entity{
Key: keyToProto(key),
}
indexedProps := 0
prevMultiple := make(map[string]*pb.Property)
for _, p := range props {
val, err := interfaceToProto(p.Value)
if err != "" {
return nil, fmt.Errorf("datastore: %s for a Property with Name %q", err, p.Name)
}
if !p.NoIndex {
rVal := reflect.ValueOf(p.Value)
if rVal.Kind() == reflect.Slice && rVal.Type().Elem().Kind() != reflect.Uint8 {
indexedProps += rVal.Len()
} else {
indexedProps++
}
}
if indexedProps > maxIndexedProperties {
return nil, errors.New("datastore: too many indexed properties")
}
switch v := p.Value.(type) {
case string:
case []byte:
if len(v) > 1500 && !p.NoIndex {
return nil, fmt.Errorf("datastore: cannot index a Property with Name %q", p.Name)
}
}
val.Indexed = proto.Bool(!p.NoIndex)
if p.Multiple {
x, ok := prevMultiple[p.Name]
if !ok {
x = &pb.Property{
Name: proto.String(p.Name),
Value: &pb.Value{},
}
prevMultiple[p.Name] = x
e.Property = append(e.Property, x)
}
x.Value.ListValue = append(x.Value.ListValue, val)
} else {
e.Property = append(e.Property, &pb.Property{
Name: proto.String(p.Name),
Value: val,
})
}
}
return e, nil
}
// DoComputeEngineCreds performs a unary RPC with compute engine auth.
func DoComputeEngineCreds(tc testpb.TestServiceClient, serviceAccount, oauthScope string) {
pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
req := &testpb.SimpleRequest{
ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(),
ResponseSize: proto.Int32(int32(largeRespSize)),
Payload: pl,
FillUsername: proto.Bool(true),
FillOauthScope: proto.Bool(true),
}
reply, err := tc.UnaryCall(context.Background(), req)
if err != nil {
grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
}
user := reply.GetUsername()
scope := reply.GetOauthScope()
if user != serviceAccount {
grpclog.Fatalf("Got user name %q, want %q.", user, serviceAccount)
}
if !strings.Contains(oauthScope, scope) {
grpclog.Fatalf("Got OAuth scope %q which is NOT a substring of %q.", scope, oauthScope)
}
grpclog.Println("ComputeEngineCreds done")
}
// callNext issues a datastore_v3/Next RPC to advance a cursor, such as that
// returned by a query with more results.
func callNext(c context.Context, res *pb.QueryResult, offset, limit int32) error {
if res.Cursor == nil {
return errors.New("datastore: internal error: server did not return a cursor")
}
req := &pb.NextRequest{
Cursor: res.Cursor,
}
if limit >= 0 {
req.Count = proto.Int32(limit)
}
if offset != 0 {
req.Offset = proto.Int32(offset)
}
if res.CompiledCursor != nil {
req.Compile = proto.Bool(true)
}
res.Reset()
return internal.Call(c, "datastore_v3", "Next", req, res)
}
// DoJWTTokenCreds performs a unary RPC with JWT token auth.
func DoJWTTokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile string) {
pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize)
req := &testpb.SimpleRequest{
ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(),
ResponseSize: proto.Int32(int32(largeRespSize)),
Payload: pl,
FillUsername: proto.Bool(true),
}
reply, err := tc.UnaryCall(context.Background(), req)
if err != nil {
grpclog.Fatal("/TestService/UnaryCall RPC failed: ", err)
}
jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile)
user := reply.GetUsername()
if !strings.Contains(string(jsonKey), user) {
grpclog.Fatalf("Got user name %q which is NOT a substring of %q.", user, jsonKey)
}
grpclog.Println("JWTtokenCreds done")
}
// GetMulti is a batch version of Get. The returned map from keys to items may
// have fewer elements than the input slice, due to memcache cache misses.
// Each key must be at most 250 bytes in length.
func GetMulti(c context.Context, key []string) (map[string]*Item, error) {
if len(key) == 0 {
return nil, nil
}
keyAsBytes := make([][]byte, len(key))
for i, k := range key {
keyAsBytes[i] = []byte(k)
}
req := &pb.MemcacheGetRequest{
Key: keyAsBytes,
ForCas: proto.Bool(true),
}
res := &pb.MemcacheGetResponse{}
if err := internal.Call(c, "memcache", "Get", req, res); err != nil {
return nil, err
}
m := make(map[string]*Item, len(res.Item))
for _, p := range res.Item {
t := protoToItem(p)
m[t.Key] = t
}
return m, nil
}
func interfaceToProto(iv interface{}) (p *pb.Value, errStr string) {
val := new(pb.Value)
switch v := iv.(type) {
case int:
val.IntegerValue = proto.Int64(int64(v))
case int32:
val.IntegerValue = proto.Int64(int64(v))
case int64:
val.IntegerValue = proto.Int64(v)
case bool:
val.BooleanValue = proto.Bool(v)
case string:
val.StringValue = proto.String(v)
case float32:
val.DoubleValue = proto.Float64(float64(v))
case float64:
val.DoubleValue = proto.Float64(v)
case *Key:
if v != nil {
val.KeyValue = keyToProto(v)
}
case time.Time:
if v.Before(minTime) || v.After(maxTime) {
return nil, fmt.Sprintf("time value out of range")
}
val.TimestampMicrosecondsValue = proto.Int64(toUnixMicro(v))
case []byte:
val.BlobValue = v
default:
if iv != nil {
return nil, fmt.Sprintf("invalid Value type %t", iv)
}
}
// TODO(jbd): Support ListValue and EntityValue.
// TODO(jbd): Support types whose underlying type is one of the types above.
return val, ""
}
// toProto converts the query to a protocol buffer.
func (q *Query) toProto(dst *pb.Query, appID string) error {
if len(q.projection) != 0 && q.keysOnly {
return errors.New("datastore: query cannot both project and be keys-only")
}
dst.Reset()
dst.App = proto.String(appID)
if q.kind != "" {
dst.Kind = proto.String(q.kind)
}
if q.ancestor != nil {
dst.Ancestor = keyToProto(appID, q.ancestor)
if q.eventual {
dst.Strong = proto.Bool(false)
}
}
if q.projection != nil {
dst.PropertyName = q.projection
if q.distinct {
dst.GroupByPropertyName = q.projection
}
}
if q.keysOnly {
dst.KeysOnly = proto.Bool(true)
dst.RequirePerfectPlan = proto.Bool(true)
}
for _, qf := range q.filter {
if qf.FieldName == "" {
return errors.New("datastore: empty query filter field name")
}
p, errStr := valueToProto(appID, qf.FieldName, reflect.ValueOf(qf.Value), false)
if errStr != "" {
return errors.New("datastore: bad query filter value type: " + errStr)
}
xf := &pb.Query_Filter{
Op: operatorToProto[qf.Op],
Property: []*pb.Property{p},
}
if xf.Op == nil {
return errors.New("datastore: unknown query filter operator")
}
dst.Filter = append(dst.Filter, xf)
}
for _, qo := range q.order {
if qo.FieldName == "" {
return errors.New("datastore: empty query order field name")
}
xo := &pb.Query_Order{
Property: proto.String(qo.FieldName),
Direction: sortDirectionToProto[qo.Direction],
}
if xo.Direction == nil {
return errors.New("datastore: unknown query order direction")
}
dst.Order = append(dst.Order, xo)
}
if q.limit >= 0 {
dst.Limit = proto.Int32(q.limit)
}
if q.offset != 0 {
dst.Offset = proto.Int32(q.offset)
}
dst.CompiledCursor = q.start
dst.EndCompiledCursor = q.end
dst.Compile = proto.Bool(true)
return nil
}
// valueToProto converts a named value to a newly allocated Property.
// The returned error string is empty on success.
func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) {
var (
pv pb.PropertyValue
unsupported bool
)
switch v.Kind() {
case reflect.Invalid:
// No-op.
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
pv.Int64Value = proto.Int64(v.Int())
case reflect.Bool:
pv.BooleanValue = proto.Bool(v.Bool())
case reflect.String:
pv.StringValue = proto.String(v.String())
case reflect.Float32, reflect.Float64:
pv.DoubleValue = proto.Float64(v.Float())
case reflect.Ptr:
if k, ok := v.Interface().(*Key); ok {
if k != nil {
pv.Referencevalue = keyToReferenceValue(defaultAppID, k)
}
} else {
unsupported = true
}
case reflect.Struct:
switch t := v.Interface().(type) {
case time.Time:
if t.Before(minTime) || t.After(maxTime) {
return nil, "time value out of range"
}
pv.Int64Value = proto.Int64(toUnixMicro(t))
case appengine.GeoPoint:
if !t.Valid() {
return nil, "invalid GeoPoint value"
}
// NOTE: Strangely, latitude maps to X, longitude to Y.
pv.Pointvalue = &pb.PropertyValue_PointValue{X: &t.Lat, Y: &t.Lng}
default:
unsupported = true
}
case reflect.Slice:
if b, ok := v.Interface().([]byte); ok {
pv.StringValue = proto.String(string(b))
} else {
// nvToProto should already catch slice values.
// If we get here, we have a slice of slice values.
unsupported = true
}
default:
unsupported = true
}
if unsupported {
return nil, "unsupported datastore value type: " + v.Type().String()
}
p = &pb.Property{
Name: proto.String(name),
Value: &pv,
Multiple: proto.Bool(multiple),
}
if v.IsValid() {
switch v.Interface().(type) {
case []byte:
p.Meaning = pb.Property_BLOB.Enum()
case ByteString:
p.Meaning = pb.Property_BYTESTRING.Enum()
case appengine.BlobKey:
p.Meaning = pb.Property_BLOBKEY.Enum()
case time.Time:
p.Meaning = pb.Property_GD_WHEN.Enum()
case appengine.GeoPoint:
p.Meaning = pb.Property_GEORSS_POINT.Enum()
}
}
return p, ""
}
func propertiesToProto(defaultAppID string, key *Key, props []Property) (*pb.EntityProto, error) {
e := &pb.EntityProto{
Key: keyToProto(defaultAppID, key),
}
if key.parent == nil {
e.EntityGroup = &pb.Path{}
} else {
e.EntityGroup = keyToProto(defaultAppID, key.root()).Path
}
prevMultiple := make(map[string]bool)
for _, p := range props {
if pm, ok := prevMultiple[p.Name]; ok {
if !pm || !p.Multiple {
return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name)
}
} else {
prevMultiple[p.Name] = p.Multiple
}
x := &pb.Property{
Name: proto.String(p.Name),
Value: new(pb.PropertyValue),
Multiple: proto.Bool(p.Multiple),
}
switch v := p.Value.(type) {
case int64:
x.Value.Int64Value = proto.Int64(v)
case bool:
x.Value.BooleanValue = proto.Bool(v)
case string:
x.Value.StringValue = proto.String(v)
if p.NoIndex {
x.Meaning = pb.Property_TEXT.Enum()
}
case float64:
x.Value.DoubleValue = proto.Float64(v)
case *Key:
if v != nil {
x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v)
}
case time.Time:
if v.Before(minTime) || v.After(maxTime) {
return nil, fmt.Errorf("datastore: time value out of range")
}
x.Value.Int64Value = proto.Int64(toUnixMicro(v))
x.Meaning = pb.Property_GD_WHEN.Enum()
case appengine.BlobKey:
x.Value.StringValue = proto.String(string(v))
x.Meaning = pb.Property_BLOBKEY.Enum()
case appengine.GeoPoint:
if !v.Valid() {
return nil, fmt.Errorf("datastore: invalid GeoPoint value")
}
// NOTE: Strangely, latitude maps to X, longitude to Y.
x.Value.Pointvalue = &pb.PropertyValue_PointValue{X: &v.Lat, Y: &v.Lng}
x.Meaning = pb.Property_GEORSS_POINT.Enum()
case []byte:
x.Value.StringValue = proto.String(string(v))
x.Meaning = pb.Property_BLOB.Enum()
if !p.NoIndex {
return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name)
}
case ByteString:
x.Value.StringValue = proto.String(string(v))
x.Meaning = pb.Property_BYTESTRING.Enum()
default:
if p.Value != nil {
return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name)
}
}
if p.NoIndex {
e.RawProperty = append(e.RawProperty, x)
} else {
e.Property = append(e.Property, x)
if len(e.Property) > maxIndexedProperties {
return nil, errors.New("datastore: too many indexed properties")
}
}
}
return e, nil
}
// RoundTrip issues a single HTTP request and returns its response. Per the
// http.RoundTripper interface, RoundTrip only returns an error if there
// was an unsupported request or the URL Fetch proxy fails.
// Note that HTTP response codes such as 5xx, 403, 404, etc are not
// errors as far as the transport is concerned and will be returned
// with err set to nil.
func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) {
methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method]
if !ok {
return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method)
}
method := pb.URLFetchRequest_RequestMethod(methNum)
freq := &pb.URLFetchRequest{
Method: &method,
Url: proto.String(urlString(req.URL)),
FollowRedirects: proto.Bool(false), // http.Client's responsibility
MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate),
}
if deadline, ok := t.Context.Deadline(); ok {
freq.Deadline = proto.Float64(deadline.Sub(time.Now()).Seconds())
}
for k, vals := range req.Header {
for _, val := range vals {
freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{
Key: proto.String(k),
Value: proto.String(val),
})
}
}
if methodAcceptsRequestBody[req.Method] && req.Body != nil {
// Avoid a []byte copy if req.Body has a Bytes method.
switch b := req.Body.(type) {
case interface {
Bytes() []byte
}:
freq.Payload = b.Bytes()
default:
freq.Payload, err = ioutil.ReadAll(req.Body)
if err != nil {
return nil, err
}
}
}
fres := &pb.URLFetchResponse{}
if err := internal.Call(t.Context, "urlfetch", "Fetch", freq, fres); err != nil {
return nil, err
}
res = &http.Response{}
res.StatusCode = int(*fres.StatusCode)
res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode))
res.Header = make(http.Header)
res.Request = req
// Faked:
res.ProtoMajor = 1
res.ProtoMinor = 1
res.Proto = "HTTP/1.1"
res.Close = true
for _, h := range fres.Header {
hkey := http.CanonicalHeaderKey(*h.Key)
hval := *h.Value
if hkey == "Content-Length" {
// Will get filled in below for all but HEAD requests.
if req.Method == "HEAD" {
res.ContentLength, _ = strconv.ParseInt(hval, 10, 64)
}
continue
}
res.Header.Add(hkey, hval)
}
if req.Method != "HEAD" {
res.ContentLength = int64(len(fres.Content))
}
truncated := fres.GetContentWasTruncated()
res.Body = &bodyReader{content: fres.Content, truncated: truncated}
return
}
// set sets the given items using the given conflict resolution policy.
// appengine.MultiError may be returned.
func set(c context.Context, item []*Item, value [][]byte, policy pb.MemcacheSetRequest_SetPolicy) error {
if len(item) == 0 {
return nil
}
req := &pb.MemcacheSetRequest{
Item: make([]*pb.MemcacheSetRequest_Item, len(item)),
}
for i, t := range item {
p := &pb.MemcacheSetRequest_Item{
Key: []byte(t.Key),
}
if value == nil {
p.Value = t.Value
} else {
p.Value = value[i]
}
if t.Flags != 0 {
p.Flags = proto.Uint32(t.Flags)
}
if t.Expiration != 0 {
// In the .proto file, MemcacheSetRequest_Item uses a fixed32 (i.e. unsigned)
// for expiration time, while MemcacheGetRequest_Item uses int32 (i.e. signed).
// Throughout this .go file, we use int32.
// Also, in the proto, the expiration value is either a duration (in seconds)
// or an absolute Unix timestamp (in seconds), depending on whether the
// value is less than or greater than or equal to 30 years, respectively.
if t.Expiration < time.Second {
// Because an Expiration of 0 means no expiration, we take
// care here to translate an item with an expiration
// Duration between 0-1 seconds as immediately expiring
// (saying it expired a few seconds ago), rather than
// rounding it down to 0 and making it live forever.
p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) - 5)
} else if t.Expiration >= thirtyYears {
p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) + uint32(t.Expiration/time.Second))
} else {
p.ExpirationTime = proto.Uint32(uint32(t.Expiration / time.Second))
}
}
if t.casID != 0 {
p.CasId = proto.Uint64(t.casID)
p.ForCas = proto.Bool(true)
}
p.SetPolicy = policy.Enum()
req.Item[i] = p
}
res := &pb.MemcacheSetResponse{}
if err := internal.Call(c, "memcache", "Set", req, res); err != nil {
return err
}
if len(res.SetStatus) != len(item) {
return ErrServerError
}
me, any := make(appengine.MultiError, len(item)), false
for i, st := range res.SetStatus {
var err error
switch st {
case pb.MemcacheSetResponse_STORED:
// OK
case pb.MemcacheSetResponse_NOT_STORED:
err = ErrNotStored
case pb.MemcacheSetResponse_EXISTS:
err = ErrCASConflict
default:
err = ErrServerError
}
if err != nil {
me[i] = err
any = true
}
}
if any {
return me
}
return nil
}
