Ejemplo n.º 1
0
func lease(c appengine.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) {
	if queueName == "" {
		queueName = "default"
	}
	req := &pb.TaskQueueQueryAndOwnTasksRequest{
		QueueName:    []byte(queueName),
		LeaseSeconds: proto.Float64(float64(leaseTime)),
		MaxTasks:     proto.Int64(int64(maxTasks)),
		GroupByTag:   proto.Bool(groupByTag),
		Tag:          tag,
	}
	res := &pb.TaskQueueQueryAndOwnTasksResponse{}
	callOpts := &appengine_internal.CallOptions{
		Timeout: 10 * time.Second,
	}
	if err := c.Call("taskqueue", "QueryAndOwnTasks", req, res, callOpts); err != nil {
		return nil, err
	}
	tasks := make([]*Task, len(res.Task))
	for i, t := range res.Task {
		tasks[i] = &Task{
			Payload:    t.Body,
			Name:       string(t.TaskName),
			Method:     "PULL",
			ETA:        time.Unix(0, *t.EtaUsec*1e3),
			RetryCount: *t.RetryCount,
			Tag:        string(t.Tag),
		}
	}
	return tasks, nil
}
Ejemplo n.º 2
0
func (w *writer) Close() error {
	if w.closed {
		return fmt.Errorf("file: Writer is closed")
	}
	w.closed = true
	if err := w.bw.Flush(); err != nil {
		return err
	}
	req := &pb.CloseRequest{
		Filename: &w.filename,
		Finalize: proto.Bool(true),
	}
	res := &pb.CloseResponse{}
	return w.c.Call("file", "Close", req, res, nil)
}
Ejemplo n.º 3
0
func runOnce(c appengine.Context, f func(appengine.Context) error, opts *TransactionOptions) error {
	// Begin the transaction.
	t := &transaction{Context: c}
	req := &pb.BeginTransactionRequest{
		App: proto.String(c.FullyQualifiedAppID()),
	}
	if opts != nil && opts.XG {
		req.AllowMultipleEg = proto.Bool(true)
	}
	if err := t.Context.Call("datastore_v3", "BeginTransaction", req, &t.transaction, nil); err != nil {
		return err
	}

	// Call f, rolling back the transaction if f returns a non-nil error, or panics.
	// The panic is not recovered.
	defer func() {
		if t.finished {
			return
		}
		t.finished = true
		// Ignore the error return value, since we are already returning a non-nil
		// error (or we're panicking).
		c.Call("datastore_v3", "Rollback", &t.transaction, &pb.VoidProto{}, nil)
	}()
	if err := f(t); err != nil {
		return err
	}
	t.finished = true

	// Commit the transaction.
	res := &pb.CommitResponse{}
	err := c.Call("datastore_v3", "Commit", &t.transaction, res, nil)
	if ae, ok := err.(*appengine_internal.APIError); ok {
		if appengine.IsDevAppServer() {
			// The Python Dev AppServer raises an ApplicationError with error code 2 (which is
			// Error.CONCURRENT_TRANSACTION) and message "Concurrency exception.".
			if ae.Code == int32(pb.Error_BAD_REQUEST) && ae.Detail == "ApplicationError: 2 Concurrency exception." {
				return ErrConcurrentTransaction
			}
		}
		if ae.Code == int32(pb.Error_CONCURRENT_TRANSACTION) {
			return ErrConcurrentTransaction
		}
	}
	return err
}
Ejemplo n.º 4
0
// callNext issues a datastore_v3/Next RPC to advance a cursor, such as that
// returned by a query with more results.
func callNext(c appengine.Context, res *pb.QueryResult, offset, limit int32) error {
	if res.Cursor == nil {
		return errors.New("datastore: internal error: server did not return a cursor")
	}
	req := &pb.NextRequest{
		Cursor: res.Cursor,
	}
	if limit >= 0 {
		req.Count = proto.Int32(limit)
	}
	if offset != 0 {
		req.Offset = proto.Int32(offset)
	}
	if res.CompiledCursor != nil {
		req.Compile = proto.Bool(true)
	}
	res.Reset()
	return c.Call("datastore_v3", "Next", req, res, nil)
}
Ejemplo n.º 5
0
// GetMulti is a batch version of Get. The returned map from keys to items may
// have fewer elements than the input slice, due to memcache cache misses.
// Each key must be at most 250 bytes in length.
func GetMulti(c appengine.Context, key []string) (map[string]*Item, error) {
	keyAsBytes := make([][]byte, len(key))
	for i, k := range key {
		keyAsBytes[i] = []byte(k)
	}
	req := &pb.MemcacheGetRequest{
		Key:    keyAsBytes,
		ForCas: proto.Bool(true),
	}
	res := &pb.MemcacheGetResponse{}
	if err := c.Call("memcache", "Get", req, res, nil); err != nil {
		return nil, err
	}
	m := make(map[string]*Item, len(res.Item))
	for _, p := range res.Item {
		t := protoToItem(p)
		m[t.Key] = t
	}
	return m, nil
}
Ejemplo n.º 6
0
// Close flushes outstanding buffered writes and finalizes the blob. After
// calling Close the key can be retrieved by calling Key.
func (w *Writer) Close() (closeErr error) {
	defer func() {
		// Save the error for Key
		w.closeErr = closeErr
	}()
	if w.closed {
		return errorf("Writer is already closed")
	}
	w.closed = true
	w.flush()
	if w.writeErr != nil {
		return w.writeErr
	}
	req := &filepb.CloseRequest{
		Filename: proto.String(w.filename),
		Finalize: proto.Bool(true),
	}
	res := &filepb.CloseResponse{}
	return w.c.Call("file", "Close", req, res, nil)
}
Ejemplo n.º 7
0
// Create begins creating a new blob. The provided mimeType if non-empty
// is stored in the blob's BlobInfo in datastore, else defaults to
// application/octet-stream. The returned Writer should be written to,
// then closed, and then its Key method can be called to retrieve the
// newly-created blob key if there were no errors.
func Create(c appengine.Context, mimeType string) (*Writer, error) {
	c, _ = appengine.Namespace(c, "") // Blobstore is always in the empty string namespace
	if mimeType == "" {
		mimeType = "application/octet-stream"
	}
	req := &filepb.CreateRequest{
		Filesystem:  proto.String("blobstore"),
		ContentType: filepb.FileContentType_RAW.Enum(),
		Parameters: []*filepb.CreateRequest_Parameter{
			{
				Name:  proto.String("content_type"),
				Value: proto.String(mimeType),
			},
		},
	}
	res := &filepb.CreateResponse{}
	if err := c.Call("file", "Create", req, res, nil); err != nil {
		return nil, err
	}

	w := &Writer{
		c:        c,
		filename: *res.Filename,
	}
	if !strings.HasPrefix(w.filename, blobstoreFileDirectory) {
		return nil, errorf("unexpected filename from files service: %q", w.filename)
	}

	oreq := &filepb.OpenRequest{
		Filename:      res.Filename,
		ContentType:   filepb.FileContentType_RAW.Enum(),
		OpenMode:      filepb.OpenRequest_APPEND.Enum(),
		ExclusiveLock: proto.Bool(true),
	}
	ores := &filepb.OpenResponse{}
	if err := c.Call("file", "Open", oreq, ores, nil); err != nil {
		return nil, err
	}
	return w, nil
}
Ejemplo n.º 8
0
// Create creates a new file, opened for append.
//
// The file must be closed when done writing.
//
// The provided filename may be absolute ("/gs/bucketname/objectname")
// or may be just the filename, in which case the bucket is determined
// from opts.  The absolute filename is returned.
func Create(c appengine.Context, filename string, opts *CreateOptions) (wc io.WriteCloser, absFilename string, err error) {
	reqFilename, err := opts.requestFilename(c, filename)
	if err != nil {
		return nil, "", err
	}
	req := &pb.CreateRequest{
		Filename:    &reqFilename,
		Filesystem:  proto.String("gs"),
		ContentType: pb.FileContentType_RAW.Enum(),
		Parameters: []*pb.CreateRequest_Parameter{
			{
				Name:  proto.String("content_type"),
				Value: proto.String(opts.mime()),
			},
		},
	}
	res := &pb.CreateResponse{}
	if err := c.Call("file", "Create", req, res, nil); err != nil {
		return nil, "", err
	}

	w := &writer{
		c:        c,
		filename: *res.Filename,
	}
	w.bw = bufio.NewWriterSize(appendWriter{w}, writeBufferSize)

	oreq := &pb.OpenRequest{
		Filename:      res.Filename,
		ContentType:   pb.FileContentType_RAW.Enum(),
		OpenMode:      pb.OpenRequest_APPEND.Enum(),
		ExclusiveLock: proto.Bool(true),
	}
	ores := &pb.OpenResponse{}
	if err := c.Call("file", "Open", oreq, ores, nil); err != nil {
		return nil, "", err
	}

	return w, reqFilename, nil
}
Ejemplo n.º 9
0
// RoundTrip issues a single HTTP request and returns its response. Per the
// http.RoundTripper interface, RoundTrip only returns an error if there
// was an unsupported request or the URL Fetch proxy fails.
// Note that HTTP response codes such as 5xx, 403, 404, etc are not
// errors as far as the transport is concerned and will be returned
// with err set to nil.
func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) {
	methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method]
	if !ok {
		return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method)
	}

	method := pb.URLFetchRequest_RequestMethod(methNum)

	freq := &pb.URLFetchRequest{
		Method:                        &method,
		Url:                           proto.String(urlString(req.URL)),
		FollowRedirects:               proto.Bool(false), // http.Client's responsibility
		MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate),
	}
	opts := &appengine_internal.CallOptions{}

	if t.Deadline != 0 {
		freq.Deadline = proto.Float64(t.Deadline.Seconds())
		opts.Timeout = t.Deadline
	}

	for k, vals := range req.Header {
		for _, val := range vals {
			freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{
				Key:   proto.String(k),
				Value: proto.String(val),
			})
		}
	}
	if methodAcceptsRequestBody[req.Method] && req.Body != nil {
		freq.Payload, err = ioutil.ReadAll(req.Body)
		if err != nil {
			return nil, err
		}
	}

	fres := &pb.URLFetchResponse{}
	if err := t.Context.Call("urlfetch", "Fetch", freq, fres, opts); err != nil {
		return nil, err
	}

	res = &http.Response{}
	res.StatusCode = int(*fres.StatusCode)
	res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode))
	res.Header = make(http.Header)
	res.Request = req

	// Faked:
	res.ProtoMajor = 1
	res.ProtoMinor = 1
	res.Proto = "HTTP/1.1"
	res.Close = true

	for _, h := range fres.Header {
		hkey := http.CanonicalHeaderKey(*h.Key)
		hval := *h.Value
		if hkey == "Content-Length" {
			// Will get filled in below for all but HEAD requests.
			if req.Method == "HEAD" {
				res.ContentLength, _ = strconv.ParseInt(hval, 10, 64)
			}
			continue
		}
		res.Header.Add(hkey, hval)
	}

	if req.Method != "HEAD" {
		res.ContentLength = int64(len(fres.Content))
	}

	truncated := fres.GetContentWasTruncated()
	res.Body = &bodyReader{content: fres.Content, truncated: truncated}
	return
}
Ejemplo n.º 10
0
// set sets the given items using the given conflict resolution policy.
// appengine.MultiError may be returned.
func set(c appengine.Context, item []*Item, value [][]byte, policy pb.MemcacheSetRequest_SetPolicy) error {
	req := &pb.MemcacheSetRequest{
		Item: make([]*pb.MemcacheSetRequest_Item, len(item)),
	}
	for i, t := range item {
		p := &pb.MemcacheSetRequest_Item{
			Key: []byte(t.Key),
		}
		if value == nil {
			p.Value = t.Value
		} else {
			p.Value = value[i]
		}
		if t.Flags != 0 {
			p.Flags = proto.Uint32(t.Flags)
		}
		if t.Expiration != 0 {
			// In the .proto file, MemcacheSetRequest_Item uses a fixed32 (i.e. unsigned)
			// for expiration time, while MemcacheGetRequest_Item uses int32 (i.e. signed).
			// Throughout this .go file, we use int32.
			// Also, in the proto, the expiration value is either a duration (in seconds)
			// or an absolute Unix timestamp (in seconds), depending on whether the
			// value is less than or greater than or equal to 30 years, respectively.
			if t.Expiration < time.Second {
				// Because an Expiration of 0 means no expiration, we take
				// care here to translate an item with an expiration
				// Duration between 0-1 seconds as immediately expiring
				// (saying it expired a few seconds ago), rather than
				// rounding it down to 0 and making it live forever.
				p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) - 5)
			} else if t.Expiration >= thirtyYears {
				p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) + uint32(t.Expiration/time.Second))
			} else {
				p.ExpirationTime = proto.Uint32(uint32(t.Expiration / time.Second))
			}
		}
		if t.casID != 0 {
			p.CasId = proto.Uint64(t.casID)
			p.ForCas = proto.Bool(true)
		}
		p.SetPolicy = policy.Enum()
		req.Item[i] = p
	}
	res := &pb.MemcacheSetResponse{}
	if err := c.Call("memcache", "Set", req, res, nil); err != nil {
		return err
	}
	if len(res.SetStatus) != len(item) {
		return ErrServerError
	}
	me, any := make(appengine.MultiError, len(item)), false
	for i, st := range res.SetStatus {
		var err error
		switch st {
		case pb.MemcacheSetResponse_STORED:
			// OK
		case pb.MemcacheSetResponse_NOT_STORED:
			err = ErrNotStored
		case pb.MemcacheSetResponse_EXISTS:
			err = ErrCASConflict
		default:
			err = ErrServerError
		}
		if err != nil {
			me[i] = err
			any = true
		}
	}
	if any {
		return me
	}
	return nil
}
Ejemplo n.º 11
0
// valueToProto converts a named value to a newly allocated Property.
// The returned error string is empty on success.
func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) {
	var (
		pv          pb.PropertyValue
		unsupported bool
	)
	switch v.Kind() {
	case reflect.Invalid:
		// No-op.
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		pv.Int64Value = proto.Int64(v.Int())
	case reflect.Bool:
		pv.BooleanValue = proto.Bool(v.Bool())
	case reflect.String:
		pv.StringValue = proto.String(v.String())
	case reflect.Float32, reflect.Float64:
		pv.DoubleValue = proto.Float64(v.Float())
	case reflect.Ptr:
		if k, ok := v.Interface().(*Key); ok {
			if k != nil {
				pv.Referencevalue = keyToReferenceValue(defaultAppID, k)
			}
		} else {
			unsupported = true
		}
	case reflect.Struct:
		if t, ok := v.Interface().(time.Time); ok {
			if t.Before(minTime) || t.After(maxTime) {
				return nil, "time value out of range"
			}
			pv.Int64Value = proto.Int64(toUnixMicro(t))
		} else {
			unsupported = true
		}
	case reflect.Slice:
		if b, ok := v.Interface().([]byte); ok {
			pv.StringValue = proto.String(string(b))
		} else {
			// nvToProto should already catch slice values.
			// If we get here, we have a slice of slice values.
			unsupported = true
		}
	default:
		unsupported = true
	}
	if unsupported {
		return nil, "unsupported datastore value type: " + v.Type().String()
	}
	p = &pb.Property{
		Name:     proto.String(name),
		Value:    &pv,
		Multiple: proto.Bool(multiple),
	}
	if v.IsValid() {
		switch v.Interface().(type) {
		case []byte:
			p.Meaning = pb.Property_BLOB.Enum()
		case appengine.BlobKey:
			p.Meaning = pb.Property_BLOBKEY.Enum()
		case time.Time:
			p.Meaning = pb.Property_GD_WHEN.Enum()
		}
	}
	return p, ""
}
Ejemplo n.º 12
0
func propertiesToProto(defaultAppID string, key *Key, src <-chan Property) (*pb.EntityProto, error) {
	defer func() {
		for _ = range src {
			// Drain the src channel, if we exit early.
		}
	}()
	e := &pb.EntityProto{
		Key: keyToProto(defaultAppID, key),
	}
	if key.parent == nil {
		e.EntityGroup = &pb.Path{}
	} else {
		e.EntityGroup = keyToProto(defaultAppID, key.root()).Path
	}
	prevMultiple := make(map[string]bool)

	for p := range src {
		if pm, ok := prevMultiple[p.Name]; ok {
			if !pm || !p.Multiple {
				return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name)
			}
		} else {
			prevMultiple[p.Name] = p.Multiple
		}

		x := &pb.Property{
			Name:     proto.String(p.Name),
			Value:    new(pb.PropertyValue),
			Multiple: proto.Bool(p.Multiple),
		}
		switch v := p.Value.(type) {
		case int64:
			x.Value.Int64Value = proto.Int64(v)
		case bool:
			x.Value.BooleanValue = proto.Bool(v)
		case string:
			x.Value.StringValue = proto.String(v)
			if p.NoIndex {
				x.Meaning = pb.Property_TEXT.Enum()
			}
		case float64:
			x.Value.DoubleValue = proto.Float64(v)
		case *Key:
			if v != nil {
				x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v)
			}
		case time.Time:
			if v.Before(minTime) || v.After(maxTime) {
				return nil, fmt.Errorf("datastore: time value out of range")
			}
			x.Value.Int64Value = proto.Int64(toUnixMicro(v))
			x.Meaning = pb.Property_GD_WHEN.Enum()
		case appengine.BlobKey:
			x.Value.StringValue = proto.String(string(v))
			x.Meaning = pb.Property_BLOBKEY.Enum()
		case []byte:
			x.Value.StringValue = proto.String(string(v))
			x.Meaning = pb.Property_BLOB.Enum()
			if !p.NoIndex {
				return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name)
			}
		default:
			if p.Value != nil {
				return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name)
			}
		}

		if p.NoIndex {
			e.RawProperty = append(e.RawProperty, x)
		} else {
			e.Property = append(e.Property, x)
			if len(e.Property) > maxIndexedProperties {
				return nil, errors.New("datastore: too many indexed properties")
			}
		}
	}
	return e, nil
}
Ejemplo n.º 13
0
// toProto converts the query to a protocol buffer.
func (q *Query) toProto(dst *pb.Query, appID string) error {
	if q.kind == "" {
		if len(q.filter) != 0 {
			return errors.New("datastore: kindless query cannot have filters")
		}
		if len(q.order) != 0 {
			return errors.New("datastore: kindless query cannot have sort orders")
		}
	}
	if len(q.projection) != 0 && q.keysOnly {
		return errors.New("datastore: query cannot both project and be keys-only")
	}
	dst.Reset()
	dst.App = proto.String(appID)
	if q.kind != "" {
		dst.Kind = proto.String(q.kind)
	}
	if q.ancestor != nil {
		dst.Ancestor = keyToProto(appID, q.ancestor)
	}
	if q.projection != nil {
		dst.PropertyName = q.projection
		if q.distinct {
			dst.GroupByPropertyName = q.projection
		}
	}
	if q.keysOnly {
		dst.KeysOnly = proto.Bool(true)
		dst.RequirePerfectPlan = proto.Bool(true)
	}
	for _, qf := range q.filter {
		if qf.FieldName == "" {
			return errors.New("datastore: empty query filter field name")
		}
		p, errStr := valueToProto(appID, qf.FieldName, reflect.ValueOf(qf.Value), false)
		if errStr != "" {
			return errors.New("datastore: bad query filter value type: " + errStr)
		}
		xf := &pb.Query_Filter{
			Op:       operatorToProto[qf.Op],
			Property: []*pb.Property{p},
		}
		if xf.Op == nil {
			return errors.New("datastore: unknown query filter operator")
		}
		dst.Filter = append(dst.Filter, xf)
	}
	for _, qo := range q.order {
		if qo.FieldName == "" {
			return errors.New("datastore: empty query order field name")
		}
		xo := &pb.Query_Order{
			Property:  proto.String(qo.FieldName),
			Direction: sortDirectionToProto[qo.Direction],
		}
		if xo.Direction == nil {
			return errors.New("datastore: unknown query order direction")
		}
		dst.Order = append(dst.Order, xo)
	}
	if q.limit >= 0 {
		dst.Limit = proto.Int32(q.limit)
	}
	if q.offset != 0 {
		dst.Offset = proto.Int32(q.offset)
	}
	dst.CompiledCursor = q.start
	dst.EndCompiledCursor = q.end
	dst.Compile = proto.Bool(true)
	return nil
}