func lease(c appengine.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) { if queueName == "" { queueName = "default" } req := &pb.TaskQueueQueryAndOwnTasksRequest{ QueueName: []byte(queueName), LeaseSeconds: proto.Float64(float64(leaseTime)), MaxTasks: proto.Int64(int64(maxTasks)), GroupByTag: proto.Bool(groupByTag), Tag: tag, } res := &pb.TaskQueueQueryAndOwnTasksResponse{} callOpts := &appengine_internal.CallOptions{ Timeout: 10 * time.Second, } if err := c.Call("taskqueue", "QueryAndOwnTasks", req, res, callOpts); err != nil { return nil, err } tasks := make([]*Task, len(res.Task)) for i, t := range res.Task { tasks[i] = &Task{ Payload: t.Body, Name: string(t.TaskName), Method: "PULL", ETA: time.Unix(0, *t.EtaUsec*1e3), RetryCount: *t.RetryCount, Tag: string(t.Tag), } } return tasks, nil }
// AllocateIDs returns a range of n integer IDs with the given kind and parent // combination. kind cannot be empty; parent may be nil. The IDs in the range // returned will not be used by the datastore's automatic ID sequence generator // and may be used with NewKey without conflict. // // The range is inclusive at the low end and exclusive at the high end. In // other words, valid intIDs x satisfy low <= x && x < high. // // If no error is returned, low + n == high. func AllocateIDs(c appengine.Context, kind string, parent *Key, n int) (low, high int64, err error) { if kind == "" { return 0, 0, errors.New("datastore: AllocateIDs given an empty kind") } if n < 0 { return 0, 0, fmt.Errorf("datastore: AllocateIDs given a negative count: %d", n) } if n == 0 { return 0, 0, nil } req := &pb.AllocateIdsRequest{ ModelKey: keyToProto("", NewIncompleteKey(c, kind, parent)), Size: proto.Int64(int64(n)), } res := &pb.AllocateIdsResponse{} if err := c.Call("datastore_v3", "AllocateIds", req, res, nil); err != nil { return 0, 0, err } // The protobuf is inclusive at both ends. Idiomatic Go (e.g. slices, for loops) // is inclusive at the low end and exclusive at the high end, so we add 1. low = res.GetStart() high = res.GetEnd() + 1 if low+int64(n) != high { return 0, 0, fmt.Errorf("datastore: internal error: could not allocate %d IDs", n) } return low, high, nil }
func (r *reader) fetchFileData(off int64) error { req := &filepb.ReadRequest{ Filename: proto.String(r.filename), Pos: proto.Int64(off), MaxBytes: proto.Int64(readBufferSize), } res := &filepb.ReadResponse{} if err := r.c.Call("file", "Read", req, res, nil); err != nil { return err } if len(res.Data) == 0 { return io.EOF } r.buf, r.r, r.off = res.Data, 0, off return nil }
func (r *reader) fetchBlobData(off int64) error { req := &blobpb.FetchDataRequest{ BlobKey: proto.String(string(r.blobKey)), StartIndex: proto.Int64(off), EndIndex: proto.Int64(off + readBufferSize - 1), // EndIndex is inclusive. } res := &blobpb.FetchDataResponse{} if err := r.c.Call("blobstore", "FetchData", req, res, nil); err != nil { return err } if len(res.Data) == 0 { return io.EOF } r.buf, r.r, r.off = res.Data, 0, off return nil }
// Run starts a query for log records, which contain request and application // level log information. func (params *Query) Run(c appengine.Context) *Result { req := &pb.LogReadRequest{} appId := c.FullyQualifiedAppID() req.AppId = &appId if !params.StartTime.IsZero() { req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3) } if !params.EndTime.IsZero() { req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3) } if params.Offset != nil { var offset pb.LogOffset if err := proto.Unmarshal(params.Offset, &offset); err != nil { return &Result{context: c, err: fmt.Errorf("bad Offset: %v", err)} } req.Offset = &offset } if params.Incomplete { req.IncludeIncomplete = ¶ms.Incomplete } if params.AppLogs { req.IncludeAppLogs = ¶ms.AppLogs } if params.ApplyMinLevel { req.MinimumLogLevel = proto.Int32(int32(params.MinLevel)) } if params.Versions == nil { // If no versions were specified, default to the major version // used by this app. versionID := appengine.VersionID(c) if i := strings.Index(versionID, "."); i >= 0 { versionID = versionID[:i] } req.VersionId = []string{versionID} } else { req.VersionId = params.Versions } if params.RequestIDs != nil { ids := make([][]byte, len(params.RequestIDs)) for i, v := range params.RequestIDs { ids[i] = []byte(v) } req.RequestId = ids } return &Result{context: c, request: req} }
// ModifyLease modifies the lease of a task. // Used to request more processing time, or to abandon processing. // leaseTime is in seconds and must not be negative. func ModifyLease(c appengine.Context, task *Task, queueName string, leaseTime int) error { if queueName == "" { queueName = "default" } req := &pb.TaskQueueModifyTaskLeaseRequest{ QueueName: []byte(queueName), TaskName: []byte(task.Name), EtaUsec: proto.Int64(task.ETA.UnixNano() / 1e3), // Used to verify ownership. LeaseSeconds: proto.Float64(float64(leaseTime)), } res := &pb.TaskQueueModifyTaskLeaseResponse{} if err := c.Call("taskqueue", "ModifyTaskLease", req, res, nil); err != nil { return err } task.ETA = time.Unix(0, *res.UpdatedEtaUsec*1e3) return nil }
// toRetryParameter converts RetryOptions to pb.TaskQueueRetryParameters. func (opt *RetryOptions) toRetryParameters() *pb.TaskQueueRetryParameters { params := &pb.TaskQueueRetryParameters{} if opt.RetryLimit > 0 { params.RetryLimit = proto.Int32(opt.RetryLimit) } if opt.AgeLimit > 0 { params.AgeLimitSec = proto.Int64(int64(opt.AgeLimit.Seconds())) } if opt.MinBackoff > 0 { params.MinBackoffSec = proto.Float64(opt.MinBackoff.Seconds()) } if opt.MaxBackoff > 0 { params.MaxBackoffSec = proto.Float64(opt.MaxBackoff.Seconds()) } if opt.MaxDoublings > 0 || (opt.MaxDoublings == 0 && opt.ApplyZeroMaxDoublings) { params.MaxDoublings = proto.Int32(opt.MaxDoublings) } return params }
// valueToProto converts a named value to a newly allocated Property. // The returned error string is empty on success. func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) { var ( pv pb.PropertyValue unsupported bool ) switch v.Kind() { case reflect.Invalid: // No-op. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: pv.Int64Value = proto.Int64(v.Int()) case reflect.Bool: pv.BooleanValue = proto.Bool(v.Bool()) case reflect.String: pv.StringValue = proto.String(v.String()) case reflect.Float32, reflect.Float64: pv.DoubleValue = proto.Float64(v.Float()) case reflect.Ptr: if k, ok := v.Interface().(*Key); ok { if k != nil { pv.Referencevalue = keyToReferenceValue(defaultAppID, k) } } else { unsupported = true } case reflect.Struct: if t, ok := v.Interface().(time.Time); ok { if t.Before(minTime) || t.After(maxTime) { return nil, "time value out of range" } pv.Int64Value = proto.Int64(toUnixMicro(t)) } else { unsupported = true } case reflect.Slice: if b, ok := v.Interface().([]byte); ok { pv.StringValue = proto.String(string(b)) } else { // nvToProto should already catch slice values. // If we get here, we have a slice of slice values. unsupported = true } default: unsupported = true } if unsupported { return nil, "unsupported datastore value type: " + v.Type().String() } p = &pb.Property{ Name: proto.String(name), Value: &pv, Multiple: proto.Bool(multiple), } if v.IsValid() { switch v.Interface().(type) { case []byte: p.Meaning = pb.Property_BLOB.Enum() case appengine.BlobKey: p.Meaning = pb.Property_BLOBKEY.Enum() case time.Time: p.Meaning = pb.Property_GD_WHEN.Enum() } } return p, "" }
func propertiesToProto(defaultAppID string, key *Key, src <-chan Property) (*pb.EntityProto, error) { defer func() { for _ = range src { // Drain the src channel, if we exit early. } }() e := &pb.EntityProto{ Key: keyToProto(defaultAppID, key), } if key.parent == nil { e.EntityGroup = &pb.Path{} } else { e.EntityGroup = keyToProto(defaultAppID, key.root()).Path } prevMultiple := make(map[string]bool) for p := range src { if pm, ok := prevMultiple[p.Name]; ok { if !pm || !p.Multiple { return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name) } } else { prevMultiple[p.Name] = p.Multiple } x := &pb.Property{ Name: proto.String(p.Name), Value: new(pb.PropertyValue), Multiple: proto.Bool(p.Multiple), } switch v := p.Value.(type) { case int64: x.Value.Int64Value = proto.Int64(v) case bool: x.Value.BooleanValue = proto.Bool(v) case string: x.Value.StringValue = proto.String(v) if p.NoIndex { x.Meaning = pb.Property_TEXT.Enum() } case float64: x.Value.DoubleValue = proto.Float64(v) case *Key: if v != nil { x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v) } case time.Time: if v.Before(minTime) || v.After(maxTime) { return nil, fmt.Errorf("datastore: time value out of range") } x.Value.Int64Value = proto.Int64(toUnixMicro(v)) x.Meaning = pb.Property_GD_WHEN.Enum() case appengine.BlobKey: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BLOBKEY.Enum() case []byte: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BLOB.Enum() if !p.NoIndex { return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name) } default: if p.Value != nil { return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name) } } if p.NoIndex { e.RawProperty = append(e.RawProperty, x) } else { e.Property = append(e.Property, x) if len(e.Property) > maxIndexedProperties { return nil, errors.New("datastore: too many indexed properties") } } } return e, nil }
func newAddReq(c appengine.Context, task *Task, queueName string) (*pb.TaskQueueAddRequest, error) { if queueName == "" { queueName = "default" } eta := task.ETA if eta.IsZero() { eta = time.Now().Add(task.Delay) } else if task.Delay != 0 { panic("taskqueue: both Delay and ETA are set") } req := &pb.TaskQueueAddRequest{ QueueName: []byte(queueName), TaskName: []byte(task.Name), EtaUsec: proto.Int64(eta.UnixNano() / 1e3), } method := task.method() if method == "PULL" { // Pull-based task req.Body = task.Payload req.Mode = pb.TaskQueueMode_PULL.Enum() if task.Tag != "" { req.Tag = []byte(task.Tag) } // TODO: More fields will need to be set. } else { // HTTP-based task if v, ok := pb.TaskQueueAddRequest_RequestMethod_value[method]; ok { req.Method = pb.TaskQueueAddRequest_RequestMethod(v).Enum() } else { return nil, fmt.Errorf("taskqueue: bad method %q", method) } req.Url = []byte(task.Path) for k, vs := range task.Header { for _, v := range vs { req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ Key: []byte(k), Value: []byte(v), }) } } if method == "POST" || method == "PUT" { req.Body = task.Payload } // Namespace headers. if _, ok := task.Header[currentNamespace]; !ok { // Fetch the current namespace of this request. s := &basepb.StringProto{} c.Call("__go__", "GetNamespace", &basepb.VoidProto{}, s, nil) req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ Key: []byte(currentNamespace), Value: []byte(s.GetValue()), }) } if _, ok := task.Header[defaultNamespace]; !ok { // Fetch the X-AppEngine-Default-Namespace header of this request. s := &basepb.StringProto{} c.Call("__go__", "GetDefaultNamespace", &basepb.VoidProto{}, s, nil) if ns := s.GetValue(); ns != "" { req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ Key: []byte(defaultNamespace), Value: []byte(ns), }) } } } if task.RetryOptions != nil { req.RetryParameters = task.RetryOptions.toRetryParameters() } return req, nil }