func facetsToProto(src []Facet) ([]*pb.Facet, error) {
dst := make([]*pb.Facet, 0, len(src))
for _, f := range src {
if !validFieldName(f.Name) {
return nil, fmt.Errorf("search: invalid facet name %q", f.Name)
}
facetValue := &pb.FacetValue{}
switch x := f.Value.(type) {
case Atom:
if !utf8.ValidString(string(x)) {
return nil, fmt.Errorf("search: %q facet is invalid UTF-8: %q", f.Name, x)
}
facetValue.Type = pb.FacetValue_ATOM.Enum()
facetValue.StringValue = proto.String(string(x))
case float64:
if !validFloat(x) {
return nil, fmt.Errorf("search: numeric facet %q with invalid value %f", f.Name, x)
}
facetValue.Type = pb.FacetValue_NUMBER.Enum()
facetValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64))
default:
return nil, fmt.Errorf("search: unsupported facet type: %v", reflect.TypeOf(f.Value))
}
dst = append(dst, &pb.Facet{
Name: proto.String(f.Name),
Value: facetValue,
})
}
return dst, nil
}
// keyToProto converts a *Key to a Reference proto.
func keyToProto(defaultAppID string, k *Key) *pb.Reference {
appID := k.appID
if appID == "" {
appID = defaultAppID
}
n := 0
for i := k; i != nil; i = i.parent {
n++
}
e := make([]*pb.Path_Element, n)
for i := k; i != nil; i = i.parent {
n--
e[n] = &pb.Path_Element{
Type: &i.kind,
}
// At most one of {Name,Id} should be set.
// Neither will be set for incomplete keys.
if i.stringID != "" {
e[n].Name = &i.stringID
} else if i.intID != 0 {
e[n].Id = &i.intID
}
}
var namespace *string
if k.namespace != "" {
namespace = proto.String(k.namespace)
}
return &pb.Reference{
App: proto.String(appID),
NameSpace: namespace,
Path: &pb.Path{
Element: e,
},
}
}
func keyToProto(k *Key) *pb.Key {
if k == nil {
return nil
}
// TODO(jbd): Eliminate unrequired allocations.
path := []*pb.Key_PathElement(nil)
for {
el := &pb.Key_PathElement{
Kind: proto.String(k.kind),
}
if k.id != 0 {
el.Id = proto.Int64(k.id)
}
if k.name != "" {
el.Name = proto.String(k.name)
}
path = append([]*pb.Key_PathElement{el}, path...)
if k.parent == nil {
break
}
k = k.parent
}
key := &pb.Key{
PathElement: path,
}
if k.namespace != "" {
key.PartitionId = &pb.PartitionId{
Namespace: proto.String(k.namespace),
}
}
return key
}
func makeRequest(params *Query, appID, versionID string) (*pb.LogReadRequest, error) {
req := &pb.LogReadRequest{}
req.AppId = &appID
if !params.StartTime.IsZero() {
req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3)
}
if !params.EndTime.IsZero() {
req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3)
}
if len(params.Offset) > 0 {
var offset pb.LogOffset
if err := proto.Unmarshal(params.Offset, &offset); err != nil {
return nil, fmt.Errorf("bad Offset: %v", err)
}
req.Offset = &offset
}
if params.Incomplete {
req.IncludeIncomplete = ¶ms.Incomplete
}
if params.AppLogs {
req.IncludeAppLogs = ¶ms.AppLogs
}
if params.ApplyMinLevel {
req.MinimumLogLevel = proto.Int32(int32(params.MinLevel))
}
if params.Versions == nil {
// If no versions were specified, default to the default module at
// the major version being used by this module.
if i := strings.Index(versionID, "."); i >= 0 {
versionID = versionID[:i]
}
req.VersionId = []string{versionID}
} else {
req.ModuleVersion = make([]*pb.LogModuleVersion, 0, len(params.Versions))
for _, v := range params.Versions {
var m *string
if i := strings.Index(v, ":"); i >= 0 {
m, v = proto.String(v[:i]), v[i+1:]
}
req.ModuleVersion = append(req.ModuleVersion, &pb.LogModuleVersion{
ModuleId: m,
VersionId: proto.String(v),
})
}
}
if params.RequestIDs != nil {
ids := make([][]byte, len(params.RequestIDs))
for i, v := range params.RequestIDs {
ids[i] = []byte(v)
}
req.RequestId = ids
}
return req, nil
}
// LoginURLFederated is like LoginURL but accepts a user's OpenID identifier.
func LoginURLFederated(c context.Context, dest, identity string) (string, error) {
req := &pb.CreateLoginURLRequest{
DestinationUrl: proto.String(dest),
}
if identity != "" {
req.FederatedIdentity = proto.String(identity)
}
res := &pb.CreateLoginURLResponse{}
if err := internal.Call(c, "user", "CreateLoginURL", req, res); err != nil {
return "", err
}
return *res.LoginUrl, nil
}
func (c *remoteContext) call(ctx context.Context, service, method string, in, out proto.Message) error {
req, err := proto.Marshal(in)
if err != nil {
return fmt.Errorf("error marshalling request: %v", err)
}
remReq := &pb.Request{
ServiceName: proto.String(service),
Method: proto.String(method),
Request: req,
// NOTE(djd): RequestId is unused in the server.
}
req, err = proto.Marshal(remReq)
if err != nil {
return fmt.Errorf("proto.Marshal: %v", err)
}
// TODO(djd): Respect ctx.Deadline()?
resp, err := c.client.Post(c.url, "application/octet-stream", bytes.NewReader(req))
if err != nil {
return fmt.Errorf("error sending request: %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad response %d; body: %q", resp.StatusCode, body)
}
if err != nil {
return fmt.Errorf("failed reading response: %v", err)
}
remResp := &pb.Response{}
if err := proto.Unmarshal(body, remResp); err != nil {
return fmt.Errorf("error unmarshalling response: %v", err)
}
if ae := remResp.GetApplicationError(); ae != nil {
return &internal.APIError{
Code: ae.GetCode(),
Detail: ae.GetDetail(),
Service: service,
}
}
if remResp.Response == nil {
return fmt.Errorf("unexpected response: %s", proto.MarshalTextString(remResp))
}
return proto.Unmarshal(remResp.Response, out)
}
func propertiesToProto(key *Key, props []Property) (*pb.Entity, error) {
e := &pb.Entity{
Key: keyToProto(key),
}
indexedProps := 0
prevMultiple := make(map[string]*pb.Property)
for _, p := range props {
val, err := interfaceToProto(p.Value)
if err != "" {
return nil, fmt.Errorf("datastore: %s for a Property with Name %q", err, p.Name)
}
if !p.NoIndex {
rVal := reflect.ValueOf(p.Value)
if rVal.Kind() == reflect.Slice && rVal.Type().Elem().Kind() != reflect.Uint8 {
indexedProps += rVal.Len()
} else {
indexedProps++
}
}
if indexedProps > maxIndexedProperties {
return nil, errors.New("datastore: too many indexed properties")
}
switch v := p.Value.(type) {
case string:
case []byte:
if len(v) > 1500 && !p.NoIndex {
return nil, fmt.Errorf("datastore: cannot index a Property with Name %q", p.Name)
}
}
val.Indexed = proto.Bool(!p.NoIndex)
if p.Multiple {
x, ok := prevMultiple[p.Name]
if !ok {
x = &pb.Property{
Name: proto.String(p.Name),
Value: &pb.Value{},
}
prevMultiple[p.Name] = x
e.Property = append(e.Property, x)
}
x.Value.ListValue = append(x.Value.ListValue, val)
} else {
e.Property = append(e.Property, &pb.Property{
Name: proto.String(p.Name),
Value: val,
})
}
}
return e, nil
}
func sortToProto(sort *SortOptions, params *pb.SearchParams) error {
for _, e := range sort.Expressions {
spec := &pb.SortSpec{
SortExpression: proto.String(e.Expr),
}
if e.Reverse {
spec.SortDescending = proto.Bool(false)
}
if e.Default != nil {
switch d := e.Default.(type) {
case float64:
spec.DefaultValueNumeric = &d
case string:
spec.DefaultValueText = &d
default:
return fmt.Errorf("search: invalid Default type %T for expression %q", d, e.Expr)
}
}
params.SortSpec = append(params.SortSpec, spec)
}
spec := &pb.ScorerSpec{}
if sort.Limit > 0 {
spec.Limit = proto.Int32(int32(sort.Limit))
params.ScorerSpec = spec
}
if sort.Scorer != nil {
sort.Scorer.toProto(spec)
params.ScorerSpec = spec
}
return nil
}
// Search searches the index for the given query.
func (x *Index) Search(c context.Context, query string, opts *SearchOptions) *Iterator {
t := &Iterator{
c: c,
index: x,
searchQuery: query,
more: moreSearch,
}
if opts != nil {
if opts.Cursor != "" {
if opts.Offset != 0 {
return errIter("at most one of Cursor and Offset may be specified")
}
t.searchCursor = proto.String(string(opts.Cursor))
}
t.limit = opts.Limit
t.fields = opts.Fields
t.idsOnly = opts.IDsOnly
t.sort = opts.Sort
t.exprs = opts.Expressions
t.refinements = opts.Refinements
t.facetOpts = opts.Facets
t.searchOffset = opts.Offset
}
return t
}
// Put saves src to the index. If id is empty, a new ID is allocated by the
// service and returned. If id is not empty, any existing index entry for that
// ID is replaced.
//
// The ID is a human-readable ASCII string. It must contain no whitespace
// characters and not start with "!".
//
// src must be a non-nil struct pointer or implement the FieldLoadSaver
// interface.
func (x *Index) Put(c context.Context, id string, src interface{}) (string, error) {
d, err := saveDoc(src)
if err != nil {
return "", err
}
if id != "" {
if !validIndexNameOrDocID(id) {
return "", fmt.Errorf("search: invalid ID %q", id)
}
d.Id = proto.String(id)
}
req := &pb.IndexDocumentRequest{
Params: &pb.IndexDocumentParams{
Document: []*pb.Document{d},
IndexSpec: &x.spec,
},
}
res := &pb.IndexDocumentResponse{}
if err := internal.Call(c, "search", "IndexDocument", req, res); err != nil {
return "", err
}
if len(res.Status) > 0 {
if s := res.Status[0]; s.GetCode() != pb.SearchServiceError_OK {
return "", fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail())
}
}
if len(res.Status) != 1 || len(res.DocId) != 1 {
return "", fmt.Errorf("search: internal error: wrong number of results (%d Statuses, %d DocIDs)",
len(res.Status), len(res.DocId))
}
return res.DocId[0], nil
}
func rangeToProto(r Range) (*pb.FacetRange, error) {
rng := &pb.FacetRange{}
if r.Start != negInf {
if !validFloat(r.Start) {
return nil, errors.New("invalid value for Start")
}
rng.Start = proto.String(strconv.FormatFloat(r.Start, 'e', -1, 64))
} else if r.End == posInf {
return nil, errors.New("either Start or End must be finite")
}
if r.End != posInf {
if !validFloat(r.End) {
return nil, errors.New("invalid value for End")
}
rng.End = proto.String(strconv.FormatFloat(r.End, 'e', -1, 64))
}
return rng, nil
}
func send(c context.Context, method string, msg *Message) error {
req := &pb.MailMessage{
Sender: &msg.Sender,
To: msg.To,
Cc: msg.Cc,
Bcc: msg.Bcc,
Subject: &msg.Subject,
}
if msg.ReplyTo != "" {
req.ReplyTo = &msg.ReplyTo
}
if msg.Body != "" {
req.TextBody = &msg.Body
}
if msg.HTMLBody != "" {
req.HtmlBody = &msg.HTMLBody
}
if len(msg.Attachments) > 0 {
req.Attachment = make([]*pb.MailAttachment, len(msg.Attachments))
for i, att := range msg.Attachments {
req.Attachment[i] = &pb.MailAttachment{
FileName: proto.String(att.Name),
Data: att.Data,
}
if att.ContentID != "" {
req.Attachment[i].ContentID = proto.String(att.ContentID)
}
}
}
for key, vs := range msg.Headers {
for _, v := range vs {
req.Header = append(req.Header, &pb.MailHeader{
Name: proto.String(key),
Value: proto.String(v),
})
}
}
res := &bpb.VoidProto{}
if err := internal.Call(c, "mail", method, req, res); err != nil {
return err
}
return nil
}
// LogoutURL returns a URL that, when visited, signs the user out,
// then redirects the user to the URL specified by dest.
func LogoutURL(c context.Context, dest string) (string, error) {
req := &pb.CreateLogoutURLRequest{
DestinationUrl: proto.String(dest),
}
res := &pb.CreateLogoutURLResponse{}
if err := internal.Call(c, "user", "CreateLogoutURL", req, res); err != nil {
return "", err
}
return *res.LogoutUrl, nil
}
func (c *context) addLogLine(ll *logpb.UserAppLogLine) {
// Truncate long log lines.
// TODO(dsymonds): Check if this is still necessary.
const lim = 8 << 10
if len(*ll.Message) > lim {
suffix := fmt.Sprintf("...(length %d)", len(*ll.Message))
ll.Message = proto.String((*ll.Message)[:lim-len(suffix)] + suffix)
}
c.pendingLogs.Lock()
c.pendingLogs.lines = append(c.pendingLogs.lines, ll)
c.pendingLogs.Unlock()
}
func refinementsToProto(refinements []Facet, params *pb.SearchParams) error {
for _, r := range refinements {
ref := &pb.FacetRefinement{
Name: proto.String(r.Name),
}
switch v := r.Value.(type) {
case Atom:
ref.Value = proto.String(string(v))
case Range:
rng, err := rangeToProto(v)
if err != nil {
return fmt.Errorf("search: refinement for facet %q: %v", r.Name, err)
}
// Unfortunately there are two identical messages for identify Facet ranges.
ref.Range = &pb.FacetRefinement_Range{Start: rng.Start, End: rng.End}
default:
return fmt.Errorf("search: unsupported refinement for facet %q of type %T", r.Name, v)
}
params.FacetRefinement = append(params.FacetRefinement, ref)
}
return nil
}
func RunTransactionOnce(c netcontext.Context, f func(netcontext.Context) error, xg bool) error {
if transactionFromContext(c) != nil {
return errors.New("nested transactions are not supported")
}
// Begin the transaction.
t := &transaction{}
req := &pb.BeginTransactionRequest{
App: proto.String(FullyQualifiedAppID(c)),
}
if xg {
req.AllowMultipleEg = proto.Bool(true)
}
if err := Call(c, "datastore_v3", "BeginTransaction", req, &t.transaction); err != nil {
return err
}
// Call f, rolling back the transaction if f returns a non-nil error, or panics.
// The panic is not recovered.
defer func() {
if t.finished {
return
}
t.finished = true
// Ignore the error return value, since we are already returning a non-nil
// error (or we're panicking).
Call(c, "datastore_v3", "Rollback", &t.transaction, &basepb.VoidProto{})
}()
if err := f(withTransaction(c, t)); err != nil {
return err
}
t.finished = true
// Commit the transaction.
res := &pb.CommitResponse{}
err := Call(c, "datastore_v3", "Commit", &t.transaction, res)
if ae, ok := err.(*APIError); ok {
/* TODO: restore this conditional
if appengine.IsDevAppServer() {
*/
// The Python Dev AppServer raises an ApplicationError with error code 2 (which is
// Error.CONCURRENT_TRANSACTION) and message "Concurrency exception.".
if ae.Code == int32(pb.Error_BAD_REQUEST) && ae.Detail == "ApplicationError: 2 Concurrency exception." {
return ErrConcurrentTransaction
}
if ae.Code == int32(pb.Error_CONCURRENT_TRANSACTION) {
return ErrConcurrentTransaction
}
}
return err
}
// fetch fetches readBufferSize bytes starting at the given offset. On success,
// the data is saved as r.buf.
func (r *reader) fetch(off int64) error {
req := &blobpb.FetchDataRequest{
BlobKey: proto.String(string(r.blobKey)),
StartIndex: proto.Int64(off),
EndIndex: proto.Int64(off + readBufferSize - 1), // EndIndex is inclusive.
}
res := &blobpb.FetchDataResponse{}
if err := internal.Call(r.c, "blobstore", "FetchData", req, res); err != nil {
return err
}
if len(res.Data) == 0 {
return io.EOF
}
r.buf, r.r, r.off = res.Data, 0, off
return nil
}
// Run runs the given query in the given context.
func (c *Client) Run(ctx context.Context, q *Query) *Iterator {
if q.err != nil {
return &Iterator{err: q.err}
}
t := &Iterator{
ctx: ctx,
client: c,
limit: q.limit,
q: q,
prevCC: q.start,
}
t.req.Reset()
if ns := ctxNamespace(ctx); ns != "" {
t.req.PartitionId = &pb.PartitionId{
Namespace: proto.String(ns),
}
}
if err := q.toProto(&t.req); err != nil {
t.err = err
return t
}
if err := c.call(ctx, "runQuery", &t.req, &t.res); err != nil {
t.err = err
return t
}
b := t.res.GetBatch()
offset := q.offset - b.GetSkippedResults()
for offset > 0 && b.GetMoreResults() == pb.QueryResultBatch_NOT_FINISHED {
t.prevCC = b.GetEndCursor()
var err error
if err = callNext(t.ctx, c, &t.req, &t.res, offset, t.limit); err != nil {
t.err = err
break
}
skip := b.GetSkippedResults()
if skip < 0 {
t.err = errors.New("datastore: internal error: negative number of skipped_results")
break
}
offset -= skip
}
if offset < 0 {
t.err = errors.New("datastore: internal error: query offset was overshot")
}
return t
}
// UploadURL creates an upload URL for the form that the user will
// fill out, passing the application path to load when the POST of the
// form is completed. These URLs expire and should not be reused. The
// opts parameter may be nil.
func UploadURL(c context.Context, successPath string, opts *UploadURLOptions) (*url.URL, error) {
req := &blobpb.CreateUploadURLRequest{
SuccessPath: proto.String(successPath),
}
if opts != nil {
if n := opts.MaxUploadBytes; n != 0 {
req.MaxUploadSizeBytes = &n
}
if n := opts.MaxUploadBytesPerBlob; n != 0 {
req.MaxUploadSizePerBlobBytes = &n
}
if s := opts.StorageBucket; s != "" {
req.GsBucketName = &s
}
}
res := &blobpb.CreateUploadURLResponse{}
if err := internal.Call(c, "blobstore", "CreateUploadURL", req, res); err != nil {
return nil, err
}
return url.Parse(*res.Url)
}
// Count returns the number of results for the given query.
func (c *Client) Count(ctx context.Context, q *Query) (int, error) {
// Check that the query is well-formed.
if q.err != nil {
return 0, q.err
}
// Run a copy of the query, with keysOnly true (if we're not a projection,
// since the two are incompatible).
newQ := q.clone()
newQ.keysOnly = len(newQ.projection) == 0
req := &pb.RunQueryRequest{}
if ns := ctxNamespace(ctx); ns != "" {
req.PartitionId = &pb.PartitionId{
Namespace: proto.String(ns),
}
}
if err := newQ.toProto(req); err != nil {
return 0, err
}
res := &pb.RunQueryResponse{}
if err := c.call(ctx, "runQuery", req, res); err != nil {
return 0, err
}
var n int
b := res.Batch
for {
n += len(b.GetEntityResult())
if b.GetMoreResults() != pb.QueryResultBatch_NOT_FINISHED {
break
}
var err error
// TODO(jbd): Support count queries that have a limit and an offset.
if err = callNext(ctx, c, req, res, 0, 0); err != nil {
return 0, err
}
}
return int(n), nil
}
// Get loads the document with the given ID into dst.
//
// The ID is a human-readable ASCII string. It must be non-empty, contain no
// whitespace characters and not start with "!".
//
// dst must be a non-nil struct pointer or implement the FieldLoadSaver
// interface.
//
// ErrFieldMismatch is returned when a field is to be loaded into a different
// type than the one it was stored from, or when a field is missing or
// unexported in the destination struct. ErrFieldMismatch is only returned if
// dst is a struct pointer. It is up to the callee to decide whether this error
// is fatal, recoverable, or ignorable.
func (x *Index) Get(c context.Context, id string, dst interface{}) error {
if id == "" || !validIndexNameOrDocID(id) {
return fmt.Errorf("search: invalid ID %q", id)
}
req := &pb.ListDocumentsRequest{
Params: &pb.ListDocumentsParams{
IndexSpec: &x.spec,
StartDocId: proto.String(id),
Limit: proto.Int32(1),
},
}
res := &pb.ListDocumentsResponse{}
if err := internal.Call(c, "search", "ListDocuments", req, res); err != nil {
return err
}
if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
}
if len(res.Document) != 1 || res.Document[0].GetId() != id {
return ErrNoSuchDocument
}
return loadDoc(dst, res.Document[0], nil)
}
func interfaceToProto(iv interface{}) (p *pb.Value, errStr string) {
val := new(pb.Value)
switch v := iv.(type) {
case int:
val.IntegerValue = proto.Int64(int64(v))
case int32:
val.IntegerValue = proto.Int64(int64(v))
case int64:
val.IntegerValue = proto.Int64(v)
case bool:
val.BooleanValue = proto.Bool(v)
case string:
val.StringValue = proto.String(v)
case float32:
val.DoubleValue = proto.Float64(float64(v))
case float64:
val.DoubleValue = proto.Float64(v)
case *Key:
if v != nil {
val.KeyValue = keyToProto(v)
}
case time.Time:
if v.Before(minTime) || v.After(maxTime) {
return nil, fmt.Sprintf("time value out of range")
}
val.TimestampMicrosecondsValue = proto.Int64(toUnixMicro(v))
case []byte:
val.BlobValue = v
default:
if iv != nil {
return nil, fmt.Sprintf("invalid Value type %t", iv)
}
}
// TODO(jbd): Support ListValue and EntityValue.
// TODO(jbd): Support types whose underlying type is one of the types above.
return val, ""
}
func handle(w http.ResponseWriter, req *http.Request) {
c := appengine.NewContext(req)
u := user.Current(c)
if u == nil {
u, _ = user.CurrentOAuth(c,
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/appengine.apis",
)
}
if u == nil || !u.Admin {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusUnauthorized)
io.WriteString(w, "You must be logged in as an administrator to access this.\n")
return
}
if req.Header.Get("X-Appcfg-Api-Version") == "" {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusForbidden)
io.WriteString(w, "This request did not contain a necessary header.\n")
return
}
if req.Method != "POST" {
// Response must be YAML.
rtok := req.FormValue("rtok")
if rtok == "" {
rtok = "0"
}
w.Header().Set("Content-Type", "text/yaml; charset=utf-8")
fmt.Fprintf(w, `{app_id: %q, rtok: %q}`, internal.FullyQualifiedAppID(c), rtok)
return
}
defer req.Body.Close()
body, err := ioutil.ReadAll(req.Body)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
log.Errorf(c, "Failed reading body: %v", err)
return
}
remReq := &pb.Request{}
if err := proto.Unmarshal(body, remReq); err != nil {
w.WriteHeader(http.StatusBadRequest)
log.Errorf(c, "Bad body: %v", err)
return
}
service, method := *remReq.ServiceName, *remReq.Method
if !requestSupported(service, method) {
w.WriteHeader(http.StatusBadRequest)
log.Errorf(c, "Unsupported RPC /%s.%s", service, method)
return
}
rawReq := &rawMessage{remReq.Request}
rawRes := &rawMessage{}
err = internal.Call(c, service, method, rawReq, rawRes)
remRes := &pb.Response{}
if err == nil {
remRes.Response = rawRes.buf
} else if ae, ok := err.(*internal.APIError); ok {
remRes.ApplicationError = &pb.ApplicationError{
Code: &ae.Code,
Detail: &ae.Detail,
}
} else {
// This shouldn't normally happen.
log.Errorf(c, "appengine/remote_api: Unexpected error of type %T: %v", err, err)
remRes.ApplicationError = &pb.ApplicationError{
Code: proto.Int32(0),
Detail: proto.String(err.Error()),
}
}
out, err := proto.Marshal(remRes)
if err != nil {
// This should not be possible.
w.WriteHeader(500)
log.Errorf(c, "proto.Marshal: %v", err)
return
}
log.Infof(c, "Spooling %d bytes of response to /%s.%s", len(out), service, method)
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Length", strconv.Itoa(len(out)))
w.Write(out)
}
// toProto converts the query to a protocol buffer.
func (q *Query) toProto(dst *pb.Query, appID string) error {
if len(q.projection) != 0 && q.keysOnly {
return errors.New("datastore: query cannot both project and be keys-only")
}
dst.Reset()
dst.App = proto.String(appID)
if q.kind != "" {
dst.Kind = proto.String(q.kind)
}
if q.ancestor != nil {
dst.Ancestor = keyToProto(appID, q.ancestor)
if q.eventual {
dst.Strong = proto.Bool(false)
}
}
if q.projection != nil {
dst.PropertyName = q.projection
if q.distinct {
dst.GroupByPropertyName = q.projection
}
}
if q.keysOnly {
dst.KeysOnly = proto.Bool(true)
dst.RequirePerfectPlan = proto.Bool(true)
}
for _, qf := range q.filter {
if qf.FieldName == "" {
return errors.New("datastore: empty query filter field name")
}
p, errStr := valueToProto(appID, qf.FieldName, reflect.ValueOf(qf.Value), false)
if errStr != "" {
return errors.New("datastore: bad query filter value type: " + errStr)
}
xf := &pb.Query_Filter{
Op: operatorToProto[qf.Op],
Property: []*pb.Property{p},
}
if xf.Op == nil {
return errors.New("datastore: unknown query filter operator")
}
dst.Filter = append(dst.Filter, xf)
}
for _, qo := range q.order {
if qo.FieldName == "" {
return errors.New("datastore: empty query order field name")
}
xo := &pb.Query_Order{
Property: proto.String(qo.FieldName),
Direction: sortDirectionToProto[qo.Direction],
}
if xo.Direction == nil {
return errors.New("datastore: unknown query order direction")
}
dst.Order = append(dst.Order, xo)
}
if q.limit >= 0 {
dst.Limit = proto.Int32(q.limit)
}
if q.offset != 0 {
dst.Offset = proto.Int32(q.offset)
}
dst.CompiledCursor = q.start
dst.EndCompiledCursor = q.end
dst.Compile = proto.Bool(true)
return nil
}
func moreSearch(t *Iterator) error {
// We use per-result (rather than single/per-page) cursors since this
// lets us return a Cursor for every iterator document. The two cursor
// types are largely interchangeable: a page cursor is the same as the
// last per-result cursor in a given search response.
req := &pb.SearchRequest{
Params: &pb.SearchParams{
IndexSpec: &t.index.spec,
Query: &t.searchQuery,
Cursor: t.searchCursor,
CursorType: pb.SearchParams_PER_RESULT.Enum(),
FieldSpec: &pb.FieldSpec{
Name: t.fields,
},
},
}
if t.limit > 0 {
req.Params.Limit = proto.Int32(int32(t.limit))
}
if t.searchOffset > 0 {
req.Params.Offset = proto.Int32(int32(t.searchOffset))
t.searchOffset = 0
}
if t.idsOnly {
req.Params.KeysOnly = &t.idsOnly
}
if t.sort != nil {
if err := sortToProto(t.sort, req.Params); err != nil {
return err
}
}
if t.refinements != nil {
if err := refinementsToProto(t.refinements, req.Params); err != nil {
return err
}
}
for _, e := range t.exprs {
req.Params.FieldSpec.Expression = append(req.Params.FieldSpec.Expression, &pb.FieldSpec_Expression{
Name: proto.String(e.Name),
Expression: proto.String(e.Expr),
})
}
for _, f := range t.facetOpts {
if err := f.setParams(req.Params); err != nil {
return fmt.Errorf("bad FacetSearchOption: %v", err)
}
}
// Don't repeat facet search.
t.facetOpts = nil
res := &pb.SearchResponse{}
if err := internal.Call(t.c, "search", "Search", req, res); err != nil {
return err
}
if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
}
t.searchRes = res.Result
if len(res.FacetResult) > 0 {
t.facetRes = res.FacetResult
}
t.count = int(*res.MatchedCount)
if t.limit > 0 {
t.more = nil
} else {
t.more = moreSearch
}
return nil
}
func fieldsToProto(src []Field) ([]*pb.Field, error) {
// Maps to catch duplicate time or numeric fields.
timeFields, numericFields := make(map[string]bool), make(map[string]bool)
dst := make([]*pb.Field, 0, len(src))
for _, f := range src {
if !validFieldName(f.Name) {
return nil, fmt.Errorf("search: invalid field name %q", f.Name)
}
fieldValue := &pb.FieldValue{}
switch x := f.Value.(type) {
case string:
fieldValue.Type = pb.FieldValue_TEXT.Enum()
fieldValue.StringValue = proto.String(x)
case Atom:
fieldValue.Type = pb.FieldValue_ATOM.Enum()
fieldValue.StringValue = proto.String(string(x))
case HTML:
fieldValue.Type = pb.FieldValue_HTML.Enum()
fieldValue.StringValue = proto.String(string(x))
case time.Time:
if timeFields[f.Name] {
return nil, fmt.Errorf("search: duplicate time field %q", f.Name)
}
timeFields[f.Name] = true
fieldValue.Type = pb.FieldValue_DATE.Enum()
fieldValue.StringValue = proto.String(strconv.FormatInt(x.UnixNano()/1e6, 10))
case float64:
if numericFields[f.Name] {
return nil, fmt.Errorf("search: duplicate numeric field %q", f.Name)
}
if !validFloat(x) {
return nil, fmt.Errorf("search: numeric field %q with invalid value %f", f.Name, x)
}
numericFields[f.Name] = true
fieldValue.Type = pb.FieldValue_NUMBER.Enum()
fieldValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64))
case appengine.GeoPoint:
if !x.Valid() {
return nil, fmt.Errorf(
"search: GeoPoint field %q with invalid value %v",
f.Name, x)
}
fieldValue.Type = pb.FieldValue_GEO.Enum()
fieldValue.Geo = &pb.FieldValue_Geo{
Lat: proto.Float64(x.Lat),
Lng: proto.Float64(x.Lng),
}
default:
return nil, fmt.Errorf("search: unsupported field type: %v", reflect.TypeOf(f.Value))
}
if f.Language != "" {
switch f.Value.(type) {
case string, HTML:
if !validLanguage(f.Language) {
return nil, fmt.Errorf("search: invalid language for field %q: %q", f.Name, f.Language)
}
fieldValue.Language = proto.String(f.Language)
default:
return nil, fmt.Errorf("search: setting language not supported for field %q of type %T", f.Name, f.Value)
}
}
if p := fieldValue.StringValue; p != nil && !utf8.ValidString(*p) {
return nil, fmt.Errorf("search: %q field is invalid UTF-8: %q", f.Name, *p)
}
dst = append(dst, &pb.Field{
Name: proto.String(f.Name),
Value: fieldValue,
})
}
return dst, nil
}
// RoundTrip issues a single HTTP request and returns its response. Per the
// http.RoundTripper interface, RoundTrip only returns an error if there
// was an unsupported request or the URL Fetch proxy fails.
// Note that HTTP response codes such as 5xx, 403, 404, etc are not
// errors as far as the transport is concerned and will be returned
// with err set to nil.
func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) {
methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method]
if !ok {
return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method)
}
method := pb.URLFetchRequest_RequestMethod(methNum)
freq := &pb.URLFetchRequest{
Method: &method,
Url: proto.String(urlString(req.URL)),
FollowRedirects: proto.Bool(false), // http.Client's responsibility
MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate),
}
if deadline, ok := t.Context.Deadline(); ok {
freq.Deadline = proto.Float64(deadline.Sub(time.Now()).Seconds())
}
for k, vals := range req.Header {
for _, val := range vals {
freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{
Key: proto.String(k),
Value: proto.String(val),
})
}
}
if methodAcceptsRequestBody[req.Method] && req.Body != nil {
// Avoid a []byte copy if req.Body has a Bytes method.
switch b := req.Body.(type) {
case interface {
Bytes() []byte
}:
freq.Payload = b.Bytes()
default:
freq.Payload, err = ioutil.ReadAll(req.Body)
if err != nil {
return nil, err
}
}
}
fres := &pb.URLFetchResponse{}
if err := internal.Call(t.Context, "urlfetch", "Fetch", freq, fres); err != nil {
return nil, err
}
res = &http.Response{}
res.StatusCode = int(*fres.StatusCode)
res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode))
res.Header = make(http.Header)
res.Request = req
// Faked:
res.ProtoMajor = 1
res.ProtoMinor = 1
res.Proto = "HTTP/1.1"
res.Close = true
for _, h := range fres.Header {
hkey := http.CanonicalHeaderKey(*h.Key)
hval := *h.Value
if hkey == "Content-Length" {
// Will get filled in below for all but HEAD requests.
if req.Method == "HEAD" {
res.ContentLength, _ = strconv.ParseInt(hval, 10, 64)
}
continue
}
res.Header.Add(hkey, hval)
}
if req.Method != "HEAD" {
res.ContentLength = int64(len(fres.Content))
}
truncated := fres.GetContentWasTruncated()
res.Body = &bodyReader{content: fres.Content, truncated: truncated}
return
}
// toProto converts the query to a protocol buffer.
func (q *Query) toProto(req *pb.RunQueryRequest) error {
dst := pb.Query{}
if len(q.projection) != 0 && q.keysOnly {
return errors.New("datastore: query cannot both project and be keys-only")
}
dst.Reset()
if q.kind != "" {
dst.Kind = []*pb.KindExpression{&pb.KindExpression{Name: proto.String(q.kind)}}
}
if q.projection != nil {
for _, propertyName := range q.projection {
dst.Projection = append(dst.Projection, &pb.PropertyExpression{Property: &pb.PropertyReference{Name: proto.String(propertyName)}})
}
if q.distinct {
for _, propertyName := range q.projection {
dst.GroupBy = append(dst.GroupBy, &pb.PropertyReference{Name: proto.String(propertyName)})
}
}
}
if q.keysOnly {
dst.Projection = []*pb.PropertyExpression{&pb.PropertyExpression{Property: &pb.PropertyReference{Name: proto.String(keyFieldName)}}}
}
var filters []*pb.Filter
for _, qf := range q.filter {
if qf.FieldName == "" {
return errors.New("datastore: empty query filter field name")
}
v, errStr := interfaceToProto(reflect.ValueOf(qf.Value).Interface())
if errStr != "" {
return errors.New("datastore: bad query filter value type: " + errStr)
}
xf := &pb.PropertyFilter{
Operator: operatorToProto[qf.Op],
Property: &pb.PropertyReference{Name: proto.String(qf.FieldName)},
Value: v,
}
if xf.Operator == nil {
return errors.New("datastore: unknown query filter operator")
}
filters = append(filters, &pb.Filter{PropertyFilter: xf})
}
if q.ancestor != nil {
filters = append(filters, &pb.Filter{
PropertyFilter: &pb.PropertyFilter{
Property: &pb.PropertyReference{Name: proto.String("__key__")},
Operator: pb.PropertyFilter_HAS_ANCESTOR.Enum(),
Value: &pb.Value{KeyValue: keyToProto(q.ancestor)},
}})
}
if len(filters) == 1 {
dst.Filter = filters[0]
} else if len(filters) > 1 {
dst.Filter = &pb.Filter{CompositeFilter: &pb.CompositeFilter{
Operator: pb.CompositeFilter_AND.Enum(),
Filter: filters,
}}
}
for _, qo := range q.order {
if qo.FieldName == "" {
return errors.New("datastore: empty query order field name")
}
xo := &pb.PropertyOrder{
Property: &pb.PropertyReference{Name: proto.String(qo.FieldName)},
Direction: sortDirectionToProto[qo.Direction],
}
if xo.Direction == nil {
return errors.New("datastore: unknown query order direction")
}
dst.Order = append(dst.Order, xo)
}
if q.limit >= 0 {
dst.Limit = proto.Int32(q.limit)
}
if q.offset != 0 {
dst.Offset = proto.Int32(q.offset)
}
dst.StartCursor = q.start
dst.EndCursor = q.end
if t := q.trans; t != nil {
if t.id == nil {
return errExpiredTransaction
}
req.ReadOptions = &pb.ReadOptions{Transaction: t.id}
}
req.Query = &dst
return nil
}
// valueToProto converts a named value to a newly allocated Property.
// The returned error string is empty on success.
func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) {
var (
pv pb.PropertyValue
unsupported bool
)
switch v.Kind() {
case reflect.Invalid:
// No-op.
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
pv.Int64Value = proto.Int64(v.Int())
case reflect.Bool:
pv.BooleanValue = proto.Bool(v.Bool())
case reflect.String:
pv.StringValue = proto.String(v.String())
case reflect.Float32, reflect.Float64:
pv.DoubleValue = proto.Float64(v.Float())
case reflect.Ptr:
if k, ok := v.Interface().(*Key); ok {
if k != nil {
pv.Referencevalue = keyToReferenceValue(defaultAppID, k)
}
} else {
unsupported = true
}
case reflect.Struct:
switch t := v.Interface().(type) {
case time.Time:
if t.Before(minTime) || t.After(maxTime) {
return nil, "time value out of range"
}
pv.Int64Value = proto.Int64(toUnixMicro(t))
case appengine.GeoPoint:
if !t.Valid() {
return nil, "invalid GeoPoint value"
}
// NOTE: Strangely, latitude maps to X, longitude to Y.
pv.Pointvalue = &pb.PropertyValue_PointValue{X: &t.Lat, Y: &t.Lng}
default:
unsupported = true
}
case reflect.Slice:
if b, ok := v.Interface().([]byte); ok {
pv.StringValue = proto.String(string(b))
} else {
// nvToProto should already catch slice values.
// If we get here, we have a slice of slice values.
unsupported = true
}
default:
unsupported = true
}
if unsupported {
return nil, "unsupported datastore value type: " + v.Type().String()
}
p = &pb.Property{
Name: proto.String(name),
Value: &pv,
Multiple: proto.Bool(multiple),
}
if v.IsValid() {
switch v.Interface().(type) {
case []byte:
p.Meaning = pb.Property_BLOB.Enum()
case ByteString:
p.Meaning = pb.Property_BYTESTRING.Enum()
case appengine.BlobKey:
p.Meaning = pb.Property_BLOBKEY.Enum()
case time.Time:
p.Meaning = pb.Property_GD_WHEN.Enum()
case appengine.GeoPoint:
p.Meaning = pb.Property_GEORSS_POINT.Enum()
}
}
return p, ""
}
func propertiesToProto(defaultAppID string, key *Key, props []Property) (*pb.EntityProto, error) {
e := &pb.EntityProto{
Key: keyToProto(defaultAppID, key),
}
if key.parent == nil {
e.EntityGroup = &pb.Path{}
} else {
e.EntityGroup = keyToProto(defaultAppID, key.root()).Path
}
prevMultiple := make(map[string]bool)
for _, p := range props {
if pm, ok := prevMultiple[p.Name]; ok {
if !pm || !p.Multiple {
return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name)
}
} else {
prevMultiple[p.Name] = p.Multiple
}
x := &pb.Property{
Name: proto.String(p.Name),
Value: new(pb.PropertyValue),
Multiple: proto.Bool(p.Multiple),
}
switch v := p.Value.(type) {
case int64:
x.Value.Int64Value = proto.Int64(v)
case bool:
x.Value.BooleanValue = proto.Bool(v)
case string:
x.Value.StringValue = proto.String(v)
if p.NoIndex {
x.Meaning = pb.Property_TEXT.Enum()
}
case float64:
x.Value.DoubleValue = proto.Float64(v)
case *Key:
if v != nil {
x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v)
}
case time.Time:
if v.Before(minTime) || v.After(maxTime) {
return nil, fmt.Errorf("datastore: time value out of range")
}
x.Value.Int64Value = proto.Int64(toUnixMicro(v))
x.Meaning = pb.Property_GD_WHEN.Enum()
case appengine.BlobKey:
x.Value.StringValue = proto.String(string(v))
x.Meaning = pb.Property_BLOBKEY.Enum()
case appengine.GeoPoint:
if !v.Valid() {
return nil, fmt.Errorf("datastore: invalid GeoPoint value")
}
// NOTE: Strangely, latitude maps to X, longitude to Y.
x.Value.Pointvalue = &pb.PropertyValue_PointValue{X: &v.Lat, Y: &v.Lng}
x.Meaning = pb.Property_GEORSS_POINT.Enum()
case []byte:
x.Value.StringValue = proto.String(string(v))
x.Meaning = pb.Property_BLOB.Enum()
if !p.NoIndex {
return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name)
}
case ByteString:
x.Value.StringValue = proto.String(string(v))
x.Meaning = pb.Property_BYTESTRING.Enum()
default:
if p.Value != nil {
return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name)
}
}
if p.NoIndex {
e.RawProperty = append(e.RawProperty, x)
} else {
e.Property = append(e.Property, x)
if len(e.Property) > maxIndexedProperties {
return nil, errors.New("datastore: too many indexed properties")
}
}
}
return e, nil
}