Example #1
0
func facetsToProto(src []Facet) ([]*pb.Facet, error) {
	dst := make([]*pb.Facet, 0, len(src))
	for _, f := range src {
		if !validFieldName(f.Name) {
			return nil, fmt.Errorf("search: invalid facet name %q", f.Name)
		}
		facetValue := &pb.FacetValue{}
		switch x := f.Value.(type) {
		case Atom:
			if !utf8.ValidString(string(x)) {
				return nil, fmt.Errorf("search: %q facet is invalid UTF-8: %q", f.Name, x)
			}
			facetValue.Type = pb.FacetValue_ATOM.Enum()
			facetValue.StringValue = proto.String(string(x))
		case float64:
			if !validFloat(x) {
				return nil, fmt.Errorf("search: numeric facet %q with invalid value %f", f.Name, x)
			}
			facetValue.Type = pb.FacetValue_NUMBER.Enum()
			facetValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64))
		default:
			return nil, fmt.Errorf("search: unsupported facet type: %v", reflect.TypeOf(f.Value))
		}
		dst = append(dst, &pb.Facet{
			Name:  proto.String(f.Name),
			Value: facetValue,
		})
	}
	return dst, nil
}
Example #2
0
// keyToProto converts a *Key to a Reference proto.
func keyToProto(defaultAppID string, k *Key) *pb.Reference {
	appID := k.appID
	if appID == "" {
		appID = defaultAppID
	}
	n := 0
	for i := k; i != nil; i = i.parent {
		n++
	}
	e := make([]*pb.Path_Element, n)
	for i := k; i != nil; i = i.parent {
		n--
		e[n] = &pb.Path_Element{
			Type: &i.kind,
		}
		// At most one of {Name,Id} should be set.
		// Neither will be set for incomplete keys.
		if i.stringID != "" {
			e[n].Name = &i.stringID
		} else if i.intID != 0 {
			e[n].Id = &i.intID
		}
	}
	var namespace *string
	if k.namespace != "" {
		namespace = proto.String(k.namespace)
	}
	return &pb.Reference{
		App:       proto.String(appID),
		NameSpace: namespace,
		Path: &pb.Path{
			Element: e,
		},
	}
}
Example #3
0
func keyToProto(k *Key) *pb.Key {
	if k == nil {
		return nil
	}

	// TODO(jbd): Eliminate unrequired allocations.
	path := []*pb.Key_PathElement(nil)
	for {
		el := &pb.Key_PathElement{
			Kind: proto.String(k.kind),
		}
		if k.id != 0 {
			el.Id = proto.Int64(k.id)
		}
		if k.name != "" {
			el.Name = proto.String(k.name)
		}
		path = append([]*pb.Key_PathElement{el}, path...)
		if k.parent == nil {
			break
		}
		k = k.parent
	}
	key := &pb.Key{
		PathElement: path,
	}
	if k.namespace != "" {
		key.PartitionId = &pb.PartitionId{
			Namespace: proto.String(k.namespace),
		}
	}
	return key
}
Example #4
0
func makeRequest(params *Query, appID, versionID string) (*pb.LogReadRequest, error) {
	req := &pb.LogReadRequest{}
	req.AppId = &appID
	if !params.StartTime.IsZero() {
		req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3)
	}
	if !params.EndTime.IsZero() {
		req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3)
	}
	if len(params.Offset) > 0 {
		var offset pb.LogOffset
		if err := proto.Unmarshal(params.Offset, &offset); err != nil {
			return nil, fmt.Errorf("bad Offset: %v", err)
		}
		req.Offset = &offset
	}
	if params.Incomplete {
		req.IncludeIncomplete = &params.Incomplete
	}
	if params.AppLogs {
		req.IncludeAppLogs = &params.AppLogs
	}
	if params.ApplyMinLevel {
		req.MinimumLogLevel = proto.Int32(int32(params.MinLevel))
	}
	if params.Versions == nil {
		// If no versions were specified, default to the default module at
		// the major version being used by this module.
		if i := strings.Index(versionID, "."); i >= 0 {
			versionID = versionID[:i]
		}
		req.VersionId = []string{versionID}
	} else {
		req.ModuleVersion = make([]*pb.LogModuleVersion, 0, len(params.Versions))
		for _, v := range params.Versions {
			var m *string
			if i := strings.Index(v, ":"); i >= 0 {
				m, v = proto.String(v[:i]), v[i+1:]
			}
			req.ModuleVersion = append(req.ModuleVersion, &pb.LogModuleVersion{
				ModuleId:  m,
				VersionId: proto.String(v),
			})
		}
	}
	if params.RequestIDs != nil {
		ids := make([][]byte, len(params.RequestIDs))
		for i, v := range params.RequestIDs {
			ids[i] = []byte(v)
		}
		req.RequestId = ids
	}

	return req, nil
}
Example #5
0
// LoginURLFederated is like LoginURL but accepts a user's OpenID identifier.
func LoginURLFederated(c context.Context, dest, identity string) (string, error) {
	req := &pb.CreateLoginURLRequest{
		DestinationUrl: proto.String(dest),
	}
	if identity != "" {
		req.FederatedIdentity = proto.String(identity)
	}
	res := &pb.CreateLoginURLResponse{}
	if err := internal.Call(c, "user", "CreateLoginURL", req, res); err != nil {
		return "", err
	}
	return *res.LoginUrl, nil
}
Example #6
0
func (c *remoteContext) call(ctx context.Context, service, method string, in, out proto.Message) error {
	req, err := proto.Marshal(in)
	if err != nil {
		return fmt.Errorf("error marshalling request: %v", err)
	}

	remReq := &pb.Request{
		ServiceName: proto.String(service),
		Method:      proto.String(method),
		Request:     req,
		// NOTE(djd): RequestId is unused in the server.
	}

	req, err = proto.Marshal(remReq)
	if err != nil {
		return fmt.Errorf("proto.Marshal: %v", err)
	}

	// TODO(djd): Respect ctx.Deadline()?
	resp, err := c.client.Post(c.url, "application/octet-stream", bytes.NewReader(req))
	if err != nil {
		return fmt.Errorf("error sending request: %v", err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("bad response %d; body: %q", resp.StatusCode, body)
	}
	if err != nil {
		return fmt.Errorf("failed reading response: %v", err)
	}
	remResp := &pb.Response{}
	if err := proto.Unmarshal(body, remResp); err != nil {
		return fmt.Errorf("error unmarshalling response: %v", err)
	}

	if ae := remResp.GetApplicationError(); ae != nil {
		return &internal.APIError{
			Code:    ae.GetCode(),
			Detail:  ae.GetDetail(),
			Service: service,
		}
	}

	if remResp.Response == nil {
		return fmt.Errorf("unexpected response: %s", proto.MarshalTextString(remResp))
	}

	return proto.Unmarshal(remResp.Response, out)
}
Example #7
0
func propertiesToProto(key *Key, props []Property) (*pb.Entity, error) {
	e := &pb.Entity{
		Key: keyToProto(key),
	}
	indexedProps := 0
	prevMultiple := make(map[string]*pb.Property)
	for _, p := range props {
		val, err := interfaceToProto(p.Value)
		if err != "" {
			return nil, fmt.Errorf("datastore: %s for a Property with Name %q", err, p.Name)
		}
		if !p.NoIndex {
			rVal := reflect.ValueOf(p.Value)
			if rVal.Kind() == reflect.Slice && rVal.Type().Elem().Kind() != reflect.Uint8 {
				indexedProps += rVal.Len()
			} else {
				indexedProps++
			}
		}
		if indexedProps > maxIndexedProperties {
			return nil, errors.New("datastore: too many indexed properties")
		}
		switch v := p.Value.(type) {
		case string:
		case []byte:
			if len(v) > 1500 && !p.NoIndex {
				return nil, fmt.Errorf("datastore: cannot index a Property with Name %q", p.Name)
			}
		}
		val.Indexed = proto.Bool(!p.NoIndex)
		if p.Multiple {
			x, ok := prevMultiple[p.Name]
			if !ok {
				x = &pb.Property{
					Name:  proto.String(p.Name),
					Value: &pb.Value{},
				}
				prevMultiple[p.Name] = x
				e.Property = append(e.Property, x)
			}
			x.Value.ListValue = append(x.Value.ListValue, val)
		} else {
			e.Property = append(e.Property, &pb.Property{
				Name:  proto.String(p.Name),
				Value: val,
			})
		}
	}
	return e, nil
}
Example #8
0
func sortToProto(sort *SortOptions, params *pb.SearchParams) error {
	for _, e := range sort.Expressions {
		spec := &pb.SortSpec{
			SortExpression: proto.String(e.Expr),
		}
		if e.Reverse {
			spec.SortDescending = proto.Bool(false)
		}
		if e.Default != nil {
			switch d := e.Default.(type) {
			case float64:
				spec.DefaultValueNumeric = &d
			case string:
				spec.DefaultValueText = &d
			default:
				return fmt.Errorf("search: invalid Default type %T for expression %q", d, e.Expr)
			}
		}
		params.SortSpec = append(params.SortSpec, spec)
	}

	spec := &pb.ScorerSpec{}
	if sort.Limit > 0 {
		spec.Limit = proto.Int32(int32(sort.Limit))
		params.ScorerSpec = spec
	}
	if sort.Scorer != nil {
		sort.Scorer.toProto(spec)
		params.ScorerSpec = spec
	}

	return nil
}
Example #9
0
// Search searches the index for the given query.
func (x *Index) Search(c context.Context, query string, opts *SearchOptions) *Iterator {
	t := &Iterator{
		c:           c,
		index:       x,
		searchQuery: query,
		more:        moreSearch,
	}
	if opts != nil {
		if opts.Cursor != "" {
			if opts.Offset != 0 {
				return errIter("at most one of Cursor and Offset may be specified")
			}
			t.searchCursor = proto.String(string(opts.Cursor))
		}
		t.limit = opts.Limit
		t.fields = opts.Fields
		t.idsOnly = opts.IDsOnly
		t.sort = opts.Sort
		t.exprs = opts.Expressions
		t.refinements = opts.Refinements
		t.facetOpts = opts.Facets
		t.searchOffset = opts.Offset
	}
	return t
}
Example #10
0
// Put saves src to the index. If id is empty, a new ID is allocated by the
// service and returned. If id is not empty, any existing index entry for that
// ID is replaced.
//
// The ID is a human-readable ASCII string. It must contain no whitespace
// characters and not start with "!".
//
// src must be a non-nil struct pointer or implement the FieldLoadSaver
// interface.
func (x *Index) Put(c context.Context, id string, src interface{}) (string, error) {
	d, err := saveDoc(src)
	if err != nil {
		return "", err
	}
	if id != "" {
		if !validIndexNameOrDocID(id) {
			return "", fmt.Errorf("search: invalid ID %q", id)
		}
		d.Id = proto.String(id)
	}
	req := &pb.IndexDocumentRequest{
		Params: &pb.IndexDocumentParams{
			Document:  []*pb.Document{d},
			IndexSpec: &x.spec,
		},
	}
	res := &pb.IndexDocumentResponse{}
	if err := internal.Call(c, "search", "IndexDocument", req, res); err != nil {
		return "", err
	}
	if len(res.Status) > 0 {
		if s := res.Status[0]; s.GetCode() != pb.SearchServiceError_OK {
			return "", fmt.Errorf("search: %s: %s", s.GetCode(), s.GetErrorDetail())
		}
	}
	if len(res.Status) != 1 || len(res.DocId) != 1 {
		return "", fmt.Errorf("search: internal error: wrong number of results (%d Statuses, %d DocIDs)",
			len(res.Status), len(res.DocId))
	}
	return res.DocId[0], nil
}
Example #11
0
func rangeToProto(r Range) (*pb.FacetRange, error) {
	rng := &pb.FacetRange{}
	if r.Start != negInf {
		if !validFloat(r.Start) {
			return nil, errors.New("invalid value for Start")
		}
		rng.Start = proto.String(strconv.FormatFloat(r.Start, 'e', -1, 64))
	} else if r.End == posInf {
		return nil, errors.New("either Start or End must be finite")
	}
	if r.End != posInf {
		if !validFloat(r.End) {
			return nil, errors.New("invalid value for End")
		}
		rng.End = proto.String(strconv.FormatFloat(r.End, 'e', -1, 64))
	}
	return rng, nil
}
Example #12
0
func send(c context.Context, method string, msg *Message) error {
	req := &pb.MailMessage{
		Sender:  &msg.Sender,
		To:      msg.To,
		Cc:      msg.Cc,
		Bcc:     msg.Bcc,
		Subject: &msg.Subject,
	}
	if msg.ReplyTo != "" {
		req.ReplyTo = &msg.ReplyTo
	}
	if msg.Body != "" {
		req.TextBody = &msg.Body
	}
	if msg.HTMLBody != "" {
		req.HtmlBody = &msg.HTMLBody
	}
	if len(msg.Attachments) > 0 {
		req.Attachment = make([]*pb.MailAttachment, len(msg.Attachments))
		for i, att := range msg.Attachments {
			req.Attachment[i] = &pb.MailAttachment{
				FileName: proto.String(att.Name),
				Data:     att.Data,
			}
			if att.ContentID != "" {
				req.Attachment[i].ContentID = proto.String(att.ContentID)
			}
		}
	}
	for key, vs := range msg.Headers {
		for _, v := range vs {
			req.Header = append(req.Header, &pb.MailHeader{
				Name:  proto.String(key),
				Value: proto.String(v),
			})
		}
	}
	res := &bpb.VoidProto{}
	if err := internal.Call(c, "mail", method, req, res); err != nil {
		return err
	}
	return nil
}
Example #13
0
// LogoutURL returns a URL that, when visited, signs the user out,
// then redirects the user to the URL specified by dest.
func LogoutURL(c context.Context, dest string) (string, error) {
	req := &pb.CreateLogoutURLRequest{
		DestinationUrl: proto.String(dest),
	}
	res := &pb.CreateLogoutURLResponse{}
	if err := internal.Call(c, "user", "CreateLogoutURL", req, res); err != nil {
		return "", err
	}
	return *res.LogoutUrl, nil
}
Example #14
0
func (c *context) addLogLine(ll *logpb.UserAppLogLine) {
	// Truncate long log lines.
	// TODO(dsymonds): Check if this is still necessary.
	const lim = 8 << 10
	if len(*ll.Message) > lim {
		suffix := fmt.Sprintf("...(length %d)", len(*ll.Message))
		ll.Message = proto.String((*ll.Message)[:lim-len(suffix)] + suffix)
	}

	c.pendingLogs.Lock()
	c.pendingLogs.lines = append(c.pendingLogs.lines, ll)
	c.pendingLogs.Unlock()
}
Example #15
0
func refinementsToProto(refinements []Facet, params *pb.SearchParams) error {
	for _, r := range refinements {
		ref := &pb.FacetRefinement{
			Name: proto.String(r.Name),
		}
		switch v := r.Value.(type) {
		case Atom:
			ref.Value = proto.String(string(v))
		case Range:
			rng, err := rangeToProto(v)
			if err != nil {
				return fmt.Errorf("search: refinement for facet %q: %v", r.Name, err)
			}
			// Unfortunately there are two identical messages for identify Facet ranges.
			ref.Range = &pb.FacetRefinement_Range{Start: rng.Start, End: rng.End}
		default:
			return fmt.Errorf("search: unsupported refinement for facet %q of type %T", r.Name, v)
		}
		params.FacetRefinement = append(params.FacetRefinement, ref)
	}
	return nil
}
Example #16
0
func RunTransactionOnce(c netcontext.Context, f func(netcontext.Context) error, xg bool) error {
	if transactionFromContext(c) != nil {
		return errors.New("nested transactions are not supported")
	}

	// Begin the transaction.
	t := &transaction{}
	req := &pb.BeginTransactionRequest{
		App: proto.String(FullyQualifiedAppID(c)),
	}
	if xg {
		req.AllowMultipleEg = proto.Bool(true)
	}
	if err := Call(c, "datastore_v3", "BeginTransaction", req, &t.transaction); err != nil {
		return err
	}

	// Call f, rolling back the transaction if f returns a non-nil error, or panics.
	// The panic is not recovered.
	defer func() {
		if t.finished {
			return
		}
		t.finished = true
		// Ignore the error return value, since we are already returning a non-nil
		// error (or we're panicking).
		Call(c, "datastore_v3", "Rollback", &t.transaction, &basepb.VoidProto{})
	}()
	if err := f(withTransaction(c, t)); err != nil {
		return err
	}
	t.finished = true

	// Commit the transaction.
	res := &pb.CommitResponse{}
	err := Call(c, "datastore_v3", "Commit", &t.transaction, res)
	if ae, ok := err.(*APIError); ok {
		/* TODO: restore this conditional
		if appengine.IsDevAppServer() {
		*/
		// The Python Dev AppServer raises an ApplicationError with error code 2 (which is
		// Error.CONCURRENT_TRANSACTION) and message "Concurrency exception.".
		if ae.Code == int32(pb.Error_BAD_REQUEST) && ae.Detail == "ApplicationError: 2 Concurrency exception." {
			return ErrConcurrentTransaction
		}
		if ae.Code == int32(pb.Error_CONCURRENT_TRANSACTION) {
			return ErrConcurrentTransaction
		}
	}
	return err
}
Example #17
0
// fetch fetches readBufferSize bytes starting at the given offset. On success,
// the data is saved as r.buf.
func (r *reader) fetch(off int64) error {
	req := &blobpb.FetchDataRequest{
		BlobKey:    proto.String(string(r.blobKey)),
		StartIndex: proto.Int64(off),
		EndIndex:   proto.Int64(off + readBufferSize - 1), // EndIndex is inclusive.
	}
	res := &blobpb.FetchDataResponse{}
	if err := internal.Call(r.c, "blobstore", "FetchData", req, res); err != nil {
		return err
	}
	if len(res.Data) == 0 {
		return io.EOF
	}
	r.buf, r.r, r.off = res.Data, 0, off
	return nil
}
Example #18
0
// Run runs the given query in the given context.
func (c *Client) Run(ctx context.Context, q *Query) *Iterator {
	if q.err != nil {
		return &Iterator{err: q.err}
	}
	t := &Iterator{
		ctx:    ctx,
		client: c,
		limit:  q.limit,
		q:      q,
		prevCC: q.start,
	}
	t.req.Reset()
	if ns := ctxNamespace(ctx); ns != "" {
		t.req.PartitionId = &pb.PartitionId{
			Namespace: proto.String(ns),
		}
	}
	if err := q.toProto(&t.req); err != nil {
		t.err = err
		return t
	}
	if err := c.call(ctx, "runQuery", &t.req, &t.res); err != nil {
		t.err = err
		return t
	}
	b := t.res.GetBatch()
	offset := q.offset - b.GetSkippedResults()
	for offset > 0 && b.GetMoreResults() == pb.QueryResultBatch_NOT_FINISHED {
		t.prevCC = b.GetEndCursor()
		var err error
		if err = callNext(t.ctx, c, &t.req, &t.res, offset, t.limit); err != nil {
			t.err = err
			break
		}
		skip := b.GetSkippedResults()
		if skip < 0 {
			t.err = errors.New("datastore: internal error: negative number of skipped_results")
			break
		}
		offset -= skip
	}
	if offset < 0 {
		t.err = errors.New("datastore: internal error: query offset was overshot")
	}
	return t
}
Example #19
0
// UploadURL creates an upload URL for the form that the user will
// fill out, passing the application path to load when the POST of the
// form is completed. These URLs expire and should not be reused. The
// opts parameter may be nil.
func UploadURL(c context.Context, successPath string, opts *UploadURLOptions) (*url.URL, error) {
	req := &blobpb.CreateUploadURLRequest{
		SuccessPath: proto.String(successPath),
	}
	if opts != nil {
		if n := opts.MaxUploadBytes; n != 0 {
			req.MaxUploadSizeBytes = &n
		}
		if n := opts.MaxUploadBytesPerBlob; n != 0 {
			req.MaxUploadSizePerBlobBytes = &n
		}
		if s := opts.StorageBucket; s != "" {
			req.GsBucketName = &s
		}
	}
	res := &blobpb.CreateUploadURLResponse{}
	if err := internal.Call(c, "blobstore", "CreateUploadURL", req, res); err != nil {
		return nil, err
	}
	return url.Parse(*res.Url)
}
Example #20
0
// Count returns the number of results for the given query.
func (c *Client) Count(ctx context.Context, q *Query) (int, error) {
	// Check that the query is well-formed.
	if q.err != nil {
		return 0, q.err
	}

	// Run a copy of the query, with keysOnly true (if we're not a projection,
	// since the two are incompatible).
	newQ := q.clone()
	newQ.keysOnly = len(newQ.projection) == 0
	req := &pb.RunQueryRequest{}

	if ns := ctxNamespace(ctx); ns != "" {
		req.PartitionId = &pb.PartitionId{
			Namespace: proto.String(ns),
		}
	}
	if err := newQ.toProto(req); err != nil {
		return 0, err
	}
	res := &pb.RunQueryResponse{}
	if err := c.call(ctx, "runQuery", req, res); err != nil {
		return 0, err
	}
	var n int
	b := res.Batch
	for {
		n += len(b.GetEntityResult())
		if b.GetMoreResults() != pb.QueryResultBatch_NOT_FINISHED {
			break
		}
		var err error
		// TODO(jbd): Support count queries that have a limit and an offset.
		if err = callNext(ctx, c, req, res, 0, 0); err != nil {
			return 0, err
		}
	}
	return int(n), nil
}
Example #21
0
// Get loads the document with the given ID into dst.
//
// The ID is a human-readable ASCII string. It must be non-empty, contain no
// whitespace characters and not start with "!".
//
// dst must be a non-nil struct pointer or implement the FieldLoadSaver
// interface.
//
// ErrFieldMismatch is returned when a field is to be loaded into a different
// type than the one it was stored from, or when a field is missing or
// unexported in the destination struct. ErrFieldMismatch is only returned if
// dst is a struct pointer. It is up to the callee to decide whether this error
// is fatal, recoverable, or ignorable.
func (x *Index) Get(c context.Context, id string, dst interface{}) error {
	if id == "" || !validIndexNameOrDocID(id) {
		return fmt.Errorf("search: invalid ID %q", id)
	}
	req := &pb.ListDocumentsRequest{
		Params: &pb.ListDocumentsParams{
			IndexSpec:  &x.spec,
			StartDocId: proto.String(id),
			Limit:      proto.Int32(1),
		},
	}
	res := &pb.ListDocumentsResponse{}
	if err := internal.Call(c, "search", "ListDocuments", req, res); err != nil {
		return err
	}
	if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
		return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
	}
	if len(res.Document) != 1 || res.Document[0].GetId() != id {
		return ErrNoSuchDocument
	}
	return loadDoc(dst, res.Document[0], nil)
}
Example #22
0
func interfaceToProto(iv interface{}) (p *pb.Value, errStr string) {
	val := new(pb.Value)
	switch v := iv.(type) {
	case int:
		val.IntegerValue = proto.Int64(int64(v))
	case int32:
		val.IntegerValue = proto.Int64(int64(v))
	case int64:
		val.IntegerValue = proto.Int64(v)
	case bool:
		val.BooleanValue = proto.Bool(v)
	case string:
		val.StringValue = proto.String(v)
	case float32:
		val.DoubleValue = proto.Float64(float64(v))
	case float64:
		val.DoubleValue = proto.Float64(v)
	case *Key:
		if v != nil {
			val.KeyValue = keyToProto(v)
		}
	case time.Time:
		if v.Before(minTime) || v.After(maxTime) {
			return nil, fmt.Sprintf("time value out of range")
		}
		val.TimestampMicrosecondsValue = proto.Int64(toUnixMicro(v))
	case []byte:
		val.BlobValue = v
	default:
		if iv != nil {
			return nil, fmt.Sprintf("invalid Value type %t", iv)
		}
	}
	// TODO(jbd): Support ListValue and EntityValue.
	// TODO(jbd): Support types whose underlying type is one of the types above.
	return val, ""
}
Example #23
0
func handle(w http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)

	u := user.Current(c)
	if u == nil {
		u, _ = user.CurrentOAuth(c,
			"https://www.googleapis.com/auth/cloud-platform",
			"https://www.googleapis.com/auth/appengine.apis",
		)
	}

	if u == nil || !u.Admin {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusUnauthorized)
		io.WriteString(w, "You must be logged in as an administrator to access this.\n")
		return
	}
	if req.Header.Get("X-Appcfg-Api-Version") == "" {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusForbidden)
		io.WriteString(w, "This request did not contain a necessary header.\n")
		return
	}

	if req.Method != "POST" {
		// Response must be YAML.
		rtok := req.FormValue("rtok")
		if rtok == "" {
			rtok = "0"
		}
		w.Header().Set("Content-Type", "text/yaml; charset=utf-8")
		fmt.Fprintf(w, `{app_id: %q, rtok: %q}`, internal.FullyQualifiedAppID(c), rtok)
		return
	}

	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		log.Errorf(c, "Failed reading body: %v", err)
		return
	}
	remReq := &pb.Request{}
	if err := proto.Unmarshal(body, remReq); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		log.Errorf(c, "Bad body: %v", err)
		return
	}

	service, method := *remReq.ServiceName, *remReq.Method
	if !requestSupported(service, method) {
		w.WriteHeader(http.StatusBadRequest)
		log.Errorf(c, "Unsupported RPC /%s.%s", service, method)
		return
	}

	rawReq := &rawMessage{remReq.Request}
	rawRes := &rawMessage{}
	err = internal.Call(c, service, method, rawReq, rawRes)

	remRes := &pb.Response{}
	if err == nil {
		remRes.Response = rawRes.buf
	} else if ae, ok := err.(*internal.APIError); ok {
		remRes.ApplicationError = &pb.ApplicationError{
			Code:   &ae.Code,
			Detail: &ae.Detail,
		}
	} else {
		// This shouldn't normally happen.
		log.Errorf(c, "appengine/remote_api: Unexpected error of type %T: %v", err, err)
		remRes.ApplicationError = &pb.ApplicationError{
			Code:   proto.Int32(0),
			Detail: proto.String(err.Error()),
		}
	}
	out, err := proto.Marshal(remRes)
	if err != nil {
		// This should not be possible.
		w.WriteHeader(500)
		log.Errorf(c, "proto.Marshal: %v", err)
		return
	}

	log.Infof(c, "Spooling %d bytes of response to /%s.%s", len(out), service, method)
	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Length", strconv.Itoa(len(out)))
	w.Write(out)
}
Example #24
0
// toProto converts the query to a protocol buffer.
func (q *Query) toProto(dst *pb.Query, appID string) error {
	if len(q.projection) != 0 && q.keysOnly {
		return errors.New("datastore: query cannot both project and be keys-only")
	}
	dst.Reset()
	dst.App = proto.String(appID)
	if q.kind != "" {
		dst.Kind = proto.String(q.kind)
	}
	if q.ancestor != nil {
		dst.Ancestor = keyToProto(appID, q.ancestor)
		if q.eventual {
			dst.Strong = proto.Bool(false)
		}
	}
	if q.projection != nil {
		dst.PropertyName = q.projection
		if q.distinct {
			dst.GroupByPropertyName = q.projection
		}
	}
	if q.keysOnly {
		dst.KeysOnly = proto.Bool(true)
		dst.RequirePerfectPlan = proto.Bool(true)
	}
	for _, qf := range q.filter {
		if qf.FieldName == "" {
			return errors.New("datastore: empty query filter field name")
		}
		p, errStr := valueToProto(appID, qf.FieldName, reflect.ValueOf(qf.Value), false)
		if errStr != "" {
			return errors.New("datastore: bad query filter value type: " + errStr)
		}
		xf := &pb.Query_Filter{
			Op:       operatorToProto[qf.Op],
			Property: []*pb.Property{p},
		}
		if xf.Op == nil {
			return errors.New("datastore: unknown query filter operator")
		}
		dst.Filter = append(dst.Filter, xf)
	}
	for _, qo := range q.order {
		if qo.FieldName == "" {
			return errors.New("datastore: empty query order field name")
		}
		xo := &pb.Query_Order{
			Property:  proto.String(qo.FieldName),
			Direction: sortDirectionToProto[qo.Direction],
		}
		if xo.Direction == nil {
			return errors.New("datastore: unknown query order direction")
		}
		dst.Order = append(dst.Order, xo)
	}
	if q.limit >= 0 {
		dst.Limit = proto.Int32(q.limit)
	}
	if q.offset != 0 {
		dst.Offset = proto.Int32(q.offset)
	}
	dst.CompiledCursor = q.start
	dst.EndCompiledCursor = q.end
	dst.Compile = proto.Bool(true)
	return nil
}
Example #25
0
func moreSearch(t *Iterator) error {
	// We use per-result (rather than single/per-page) cursors since this
	// lets us return a Cursor for every iterator document. The two cursor
	// types are largely interchangeable: a page cursor is the same as the
	// last per-result cursor in a given search response.
	req := &pb.SearchRequest{
		Params: &pb.SearchParams{
			IndexSpec:  &t.index.spec,
			Query:      &t.searchQuery,
			Cursor:     t.searchCursor,
			CursorType: pb.SearchParams_PER_RESULT.Enum(),
			FieldSpec: &pb.FieldSpec{
				Name: t.fields,
			},
		},
	}
	if t.limit > 0 {
		req.Params.Limit = proto.Int32(int32(t.limit))
	}
	if t.searchOffset > 0 {
		req.Params.Offset = proto.Int32(int32(t.searchOffset))
		t.searchOffset = 0
	}
	if t.idsOnly {
		req.Params.KeysOnly = &t.idsOnly
	}
	if t.sort != nil {
		if err := sortToProto(t.sort, req.Params); err != nil {
			return err
		}
	}
	if t.refinements != nil {
		if err := refinementsToProto(t.refinements, req.Params); err != nil {
			return err
		}
	}
	for _, e := range t.exprs {
		req.Params.FieldSpec.Expression = append(req.Params.FieldSpec.Expression, &pb.FieldSpec_Expression{
			Name:       proto.String(e.Name),
			Expression: proto.String(e.Expr),
		})
	}
	for _, f := range t.facetOpts {
		if err := f.setParams(req.Params); err != nil {
			return fmt.Errorf("bad FacetSearchOption: %v", err)
		}
	}
	// Don't repeat facet search.
	t.facetOpts = nil

	res := &pb.SearchResponse{}
	if err := internal.Call(t.c, "search", "Search", req, res); err != nil {
		return err
	}
	if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
		return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
	}
	t.searchRes = res.Result
	if len(res.FacetResult) > 0 {
		t.facetRes = res.FacetResult
	}
	t.count = int(*res.MatchedCount)
	if t.limit > 0 {
		t.more = nil
	} else {
		t.more = moreSearch
	}
	return nil
}
Example #26
0
func fieldsToProto(src []Field) ([]*pb.Field, error) {
	// Maps to catch duplicate time or numeric fields.
	timeFields, numericFields := make(map[string]bool), make(map[string]bool)
	dst := make([]*pb.Field, 0, len(src))
	for _, f := range src {
		if !validFieldName(f.Name) {
			return nil, fmt.Errorf("search: invalid field name %q", f.Name)
		}
		fieldValue := &pb.FieldValue{}
		switch x := f.Value.(type) {
		case string:
			fieldValue.Type = pb.FieldValue_TEXT.Enum()
			fieldValue.StringValue = proto.String(x)
		case Atom:
			fieldValue.Type = pb.FieldValue_ATOM.Enum()
			fieldValue.StringValue = proto.String(string(x))
		case HTML:
			fieldValue.Type = pb.FieldValue_HTML.Enum()
			fieldValue.StringValue = proto.String(string(x))
		case time.Time:
			if timeFields[f.Name] {
				return nil, fmt.Errorf("search: duplicate time field %q", f.Name)
			}
			timeFields[f.Name] = true
			fieldValue.Type = pb.FieldValue_DATE.Enum()
			fieldValue.StringValue = proto.String(strconv.FormatInt(x.UnixNano()/1e6, 10))
		case float64:
			if numericFields[f.Name] {
				return nil, fmt.Errorf("search: duplicate numeric field %q", f.Name)
			}
			if !validFloat(x) {
				return nil, fmt.Errorf("search: numeric field %q with invalid value %f", f.Name, x)
			}
			numericFields[f.Name] = true
			fieldValue.Type = pb.FieldValue_NUMBER.Enum()
			fieldValue.StringValue = proto.String(strconv.FormatFloat(x, 'e', -1, 64))
		case appengine.GeoPoint:
			if !x.Valid() {
				return nil, fmt.Errorf(
					"search: GeoPoint field %q with invalid value %v",
					f.Name, x)
			}
			fieldValue.Type = pb.FieldValue_GEO.Enum()
			fieldValue.Geo = &pb.FieldValue_Geo{
				Lat: proto.Float64(x.Lat),
				Lng: proto.Float64(x.Lng),
			}
		default:
			return nil, fmt.Errorf("search: unsupported field type: %v", reflect.TypeOf(f.Value))
		}
		if f.Language != "" {
			switch f.Value.(type) {
			case string, HTML:
				if !validLanguage(f.Language) {
					return nil, fmt.Errorf("search: invalid language for field %q: %q", f.Name, f.Language)
				}
				fieldValue.Language = proto.String(f.Language)
			default:
				return nil, fmt.Errorf("search: setting language not supported for field %q of type %T", f.Name, f.Value)
			}
		}
		if p := fieldValue.StringValue; p != nil && !utf8.ValidString(*p) {
			return nil, fmt.Errorf("search: %q field is invalid UTF-8: %q", f.Name, *p)
		}
		dst = append(dst, &pb.Field{
			Name:  proto.String(f.Name),
			Value: fieldValue,
		})
	}
	return dst, nil
}
Example #27
0
// RoundTrip issues a single HTTP request and returns its response. Per the
// http.RoundTripper interface, RoundTrip only returns an error if there
// was an unsupported request or the URL Fetch proxy fails.
// Note that HTTP response codes such as 5xx, 403, 404, etc are not
// errors as far as the transport is concerned and will be returned
// with err set to nil.
func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) {
	methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method]
	if !ok {
		return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method)
	}

	method := pb.URLFetchRequest_RequestMethod(methNum)

	freq := &pb.URLFetchRequest{
		Method:                        &method,
		Url:                           proto.String(urlString(req.URL)),
		FollowRedirects:               proto.Bool(false), // http.Client's responsibility
		MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate),
	}
	if deadline, ok := t.Context.Deadline(); ok {
		freq.Deadline = proto.Float64(deadline.Sub(time.Now()).Seconds())
	}

	for k, vals := range req.Header {
		for _, val := range vals {
			freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{
				Key:   proto.String(k),
				Value: proto.String(val),
			})
		}
	}
	if methodAcceptsRequestBody[req.Method] && req.Body != nil {
		// Avoid a []byte copy if req.Body has a Bytes method.
		switch b := req.Body.(type) {
		case interface {
			Bytes() []byte
		}:
			freq.Payload = b.Bytes()
		default:
			freq.Payload, err = ioutil.ReadAll(req.Body)
			if err != nil {
				return nil, err
			}
		}
	}

	fres := &pb.URLFetchResponse{}
	if err := internal.Call(t.Context, "urlfetch", "Fetch", freq, fres); err != nil {
		return nil, err
	}

	res = &http.Response{}
	res.StatusCode = int(*fres.StatusCode)
	res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode))
	res.Header = make(http.Header)
	res.Request = req

	// Faked:
	res.ProtoMajor = 1
	res.ProtoMinor = 1
	res.Proto = "HTTP/1.1"
	res.Close = true

	for _, h := range fres.Header {
		hkey := http.CanonicalHeaderKey(*h.Key)
		hval := *h.Value
		if hkey == "Content-Length" {
			// Will get filled in below for all but HEAD requests.
			if req.Method == "HEAD" {
				res.ContentLength, _ = strconv.ParseInt(hval, 10, 64)
			}
			continue
		}
		res.Header.Add(hkey, hval)
	}

	if req.Method != "HEAD" {
		res.ContentLength = int64(len(fres.Content))
	}

	truncated := fres.GetContentWasTruncated()
	res.Body = &bodyReader{content: fres.Content, truncated: truncated}
	return
}
Example #28
0
// toProto converts the query to a protocol buffer.
func (q *Query) toProto(req *pb.RunQueryRequest) error {
	dst := pb.Query{}
	if len(q.projection) != 0 && q.keysOnly {
		return errors.New("datastore: query cannot both project and be keys-only")
	}
	dst.Reset()
	if q.kind != "" {
		dst.Kind = []*pb.KindExpression{&pb.KindExpression{Name: proto.String(q.kind)}}
	}
	if q.projection != nil {
		for _, propertyName := range q.projection {
			dst.Projection = append(dst.Projection, &pb.PropertyExpression{Property: &pb.PropertyReference{Name: proto.String(propertyName)}})
		}

		if q.distinct {
			for _, propertyName := range q.projection {
				dst.GroupBy = append(dst.GroupBy, &pb.PropertyReference{Name: proto.String(propertyName)})
			}
		}
	}
	if q.keysOnly {
		dst.Projection = []*pb.PropertyExpression{&pb.PropertyExpression{Property: &pb.PropertyReference{Name: proto.String(keyFieldName)}}}
	}

	var filters []*pb.Filter
	for _, qf := range q.filter {
		if qf.FieldName == "" {
			return errors.New("datastore: empty query filter field name")
		}
		v, errStr := interfaceToProto(reflect.ValueOf(qf.Value).Interface())
		if errStr != "" {
			return errors.New("datastore: bad query filter value type: " + errStr)
		}
		xf := &pb.PropertyFilter{
			Operator: operatorToProto[qf.Op],
			Property: &pb.PropertyReference{Name: proto.String(qf.FieldName)},
			Value:    v,
		}
		if xf.Operator == nil {
			return errors.New("datastore: unknown query filter operator")
		}
		filters = append(filters, &pb.Filter{PropertyFilter: xf})
	}

	if q.ancestor != nil {
		filters = append(filters, &pb.Filter{
			PropertyFilter: &pb.PropertyFilter{
				Property: &pb.PropertyReference{Name: proto.String("__key__")},
				Operator: pb.PropertyFilter_HAS_ANCESTOR.Enum(),
				Value:    &pb.Value{KeyValue: keyToProto(q.ancestor)},
			}})
	}

	if len(filters) == 1 {
		dst.Filter = filters[0]
	} else if len(filters) > 1 {
		dst.Filter = &pb.Filter{CompositeFilter: &pb.CompositeFilter{
			Operator: pb.CompositeFilter_AND.Enum(),
			Filter:   filters,
		}}
	}

	for _, qo := range q.order {
		if qo.FieldName == "" {
			return errors.New("datastore: empty query order field name")
		}
		xo := &pb.PropertyOrder{
			Property:  &pb.PropertyReference{Name: proto.String(qo.FieldName)},
			Direction: sortDirectionToProto[qo.Direction],
		}
		if xo.Direction == nil {
			return errors.New("datastore: unknown query order direction")
		}
		dst.Order = append(dst.Order, xo)
	}
	if q.limit >= 0 {
		dst.Limit = proto.Int32(q.limit)
	}
	if q.offset != 0 {
		dst.Offset = proto.Int32(q.offset)
	}
	dst.StartCursor = q.start
	dst.EndCursor = q.end

	if t := q.trans; t != nil {
		if t.id == nil {
			return errExpiredTransaction
		}
		req.ReadOptions = &pb.ReadOptions{Transaction: t.id}
	}

	req.Query = &dst
	return nil
}
Example #29
0
// valueToProto converts a named value to a newly allocated Property.
// The returned error string is empty on success.
func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) {
	var (
		pv          pb.PropertyValue
		unsupported bool
	)
	switch v.Kind() {
	case reflect.Invalid:
		// No-op.
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		pv.Int64Value = proto.Int64(v.Int())
	case reflect.Bool:
		pv.BooleanValue = proto.Bool(v.Bool())
	case reflect.String:
		pv.StringValue = proto.String(v.String())
	case reflect.Float32, reflect.Float64:
		pv.DoubleValue = proto.Float64(v.Float())
	case reflect.Ptr:
		if k, ok := v.Interface().(*Key); ok {
			if k != nil {
				pv.Referencevalue = keyToReferenceValue(defaultAppID, k)
			}
		} else {
			unsupported = true
		}
	case reflect.Struct:
		switch t := v.Interface().(type) {
		case time.Time:
			if t.Before(minTime) || t.After(maxTime) {
				return nil, "time value out of range"
			}
			pv.Int64Value = proto.Int64(toUnixMicro(t))
		case appengine.GeoPoint:
			if !t.Valid() {
				return nil, "invalid GeoPoint value"
			}
			// NOTE: Strangely, latitude maps to X, longitude to Y.
			pv.Pointvalue = &pb.PropertyValue_PointValue{X: &t.Lat, Y: &t.Lng}
		default:
			unsupported = true
		}
	case reflect.Slice:
		if b, ok := v.Interface().([]byte); ok {
			pv.StringValue = proto.String(string(b))
		} else {
			// nvToProto should already catch slice values.
			// If we get here, we have a slice of slice values.
			unsupported = true
		}
	default:
		unsupported = true
	}
	if unsupported {
		return nil, "unsupported datastore value type: " + v.Type().String()
	}
	p = &pb.Property{
		Name:     proto.String(name),
		Value:    &pv,
		Multiple: proto.Bool(multiple),
	}
	if v.IsValid() {
		switch v.Interface().(type) {
		case []byte:
			p.Meaning = pb.Property_BLOB.Enum()
		case ByteString:
			p.Meaning = pb.Property_BYTESTRING.Enum()
		case appengine.BlobKey:
			p.Meaning = pb.Property_BLOBKEY.Enum()
		case time.Time:
			p.Meaning = pb.Property_GD_WHEN.Enum()
		case appengine.GeoPoint:
			p.Meaning = pb.Property_GEORSS_POINT.Enum()
		}
	}
	return p, ""
}
Example #30
0
func propertiesToProto(defaultAppID string, key *Key, props []Property) (*pb.EntityProto, error) {
	e := &pb.EntityProto{
		Key: keyToProto(defaultAppID, key),
	}
	if key.parent == nil {
		e.EntityGroup = &pb.Path{}
	} else {
		e.EntityGroup = keyToProto(defaultAppID, key.root()).Path
	}
	prevMultiple := make(map[string]bool)

	for _, p := range props {
		if pm, ok := prevMultiple[p.Name]; ok {
			if !pm || !p.Multiple {
				return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name)
			}
		} else {
			prevMultiple[p.Name] = p.Multiple
		}

		x := &pb.Property{
			Name:     proto.String(p.Name),
			Value:    new(pb.PropertyValue),
			Multiple: proto.Bool(p.Multiple),
		}
		switch v := p.Value.(type) {
		case int64:
			x.Value.Int64Value = proto.Int64(v)
		case bool:
			x.Value.BooleanValue = proto.Bool(v)
		case string:
			x.Value.StringValue = proto.String(v)
			if p.NoIndex {
				x.Meaning = pb.Property_TEXT.Enum()
			}
		case float64:
			x.Value.DoubleValue = proto.Float64(v)
		case *Key:
			if v != nil {
				x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v)
			}
		case time.Time:
			if v.Before(minTime) || v.After(maxTime) {
				return nil, fmt.Errorf("datastore: time value out of range")
			}
			x.Value.Int64Value = proto.Int64(toUnixMicro(v))
			x.Meaning = pb.Property_GD_WHEN.Enum()
		case appengine.BlobKey:
			x.Value.StringValue = proto.String(string(v))
			x.Meaning = pb.Property_BLOBKEY.Enum()
		case appengine.GeoPoint:
			if !v.Valid() {
				return nil, fmt.Errorf("datastore: invalid GeoPoint value")
			}
			// NOTE: Strangely, latitude maps to X, longitude to Y.
			x.Value.Pointvalue = &pb.PropertyValue_PointValue{X: &v.Lat, Y: &v.Lng}
			x.Meaning = pb.Property_GEORSS_POINT.Enum()
		case []byte:
			x.Value.StringValue = proto.String(string(v))
			x.Meaning = pb.Property_BLOB.Enum()
			if !p.NoIndex {
				return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name)
			}
		case ByteString:
			x.Value.StringValue = proto.String(string(v))
			x.Meaning = pb.Property_BYTESTRING.Enum()
		default:
			if p.Value != nil {
				return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name)
			}
		}

		if p.NoIndex {
			e.RawProperty = append(e.RawProperty, x)
		} else {
			e.Property = append(e.Property, x)
			if len(e.Property) > maxIndexedProperties {
				return nil, errors.New("datastore: too many indexed properties")
			}
		}
	}
	return e, nil
}