func makeRequest(params *Query, appID, versionID string) (*pb.LogReadRequest, error) { req := &pb.LogReadRequest{} req.AppId = &appID if !params.StartTime.IsZero() { req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3) } if !params.EndTime.IsZero() { req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3) } if len(params.Offset) > 0 { var offset pb.LogOffset if err := proto.Unmarshal(params.Offset, &offset); err != nil { return nil, fmt.Errorf("bad Offset: %v", err) } req.Offset = &offset } if params.Incomplete { req.IncludeIncomplete = ¶ms.Incomplete } if params.AppLogs { req.IncludeAppLogs = ¶ms.AppLogs } if params.ApplyMinLevel { req.MinimumLogLevel = proto.Int32(int32(params.MinLevel)) } if params.Versions == nil { // If no versions were specified, default to the default module at // the major version being used by this module. if i := strings.Index(versionID, "."); i >= 0 { versionID = versionID[:i] } req.VersionId = []string{versionID} } else { req.ModuleVersion = make([]*pb.LogModuleVersion, 0, len(params.Versions)) for _, v := range params.Versions { var m *string if i := strings.Index(v, ":"); i >= 0 { m, v = proto.String(v[:i]), v[i+1:] } req.ModuleVersion = append(req.ModuleVersion, &pb.LogModuleVersion{ ModuleId: m, VersionId: proto.String(v), }) } } if params.RequestIDs != nil { ids := make([][]byte, len(params.RequestIDs)) for i, v := range params.RequestIDs { ids[i] = []byte(v) } req.RequestId = ids } return req, nil }
func keyToProto(k *Key) *pb.Key { if k == nil { return nil } // TODO(jbd): Eliminate unrequired allocations. path := []*pb.Key_PathElement(nil) for { el := &pb.Key_PathElement{ Kind: proto.String(k.kind), } if k.id != 0 { el.Id = proto.Int64(k.id) } if k.name != "" { el.Name = proto.String(k.name) } path = append([]*pb.Key_PathElement{el}, path...) if k.parent == nil { break } k = k.parent } key := &pb.Key{ PathElement: path, } if k.namespace != "" { key.PartitionId = &pb.PartitionId{ Namespace: proto.String(k.namespace), } } return key }
// AllocateIDs returns a range of n integer IDs with the given kind and parent // combination. kind cannot be empty; parent may be nil. The IDs in the range // returned will not be used by the datastore's automatic ID sequence generator // and may be used with NewKey without conflict. // // The range is inclusive at the low end and exclusive at the high end. In // other words, valid intIDs x satisfy low <= x && x < high. // // If no error is returned, low + n == high. func AllocateIDs(c context.Context, kind string, parent *Key, n int) (low, high int64, err error) { if kind == "" { return 0, 0, errors.New("datastore: AllocateIDs given an empty kind") } if n < 0 { return 0, 0, fmt.Errorf("datastore: AllocateIDs given a negative count: %d", n) } if n == 0 { return 0, 0, nil } req := &pb.AllocateIdsRequest{ ModelKey: keyToProto("", NewIncompleteKey(c, kind, parent)), Size: proto.Int64(int64(n)), } res := &pb.AllocateIdsResponse{} if err := internal.Call(c, "datastore_v3", "AllocateIds", req, res); err != nil { return 0, 0, err } // The protobuf is inclusive at both ends. Idiomatic Go (e.g. slices, for loops) // is inclusive at the low end and exclusive at the high end, so we add 1. low = res.GetStart() high = res.GetEnd() + 1 if low+int64(n) != high { return 0, 0, fmt.Errorf("datastore: internal error: could not allocate %d IDs", n) } return low, high, nil }
func lease(c context.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) { if queueName == "" { queueName = "default" } req := &pb.TaskQueueQueryAndOwnTasksRequest{ QueueName: []byte(queueName), LeaseSeconds: proto.Float64(float64(leaseTime)), MaxTasks: proto.Int64(int64(maxTasks)), GroupByTag: proto.Bool(groupByTag), Tag: tag, } res := &pb.TaskQueueQueryAndOwnTasksResponse{} if err := internal.Call(c, "taskqueue", "QueryAndOwnTasks", req, res); err != nil { return nil, err } tasks := make([]*Task, len(res.Task)) for i, t := range res.Task { tasks[i] = &Task{ Payload: t.Body, Name: string(t.TaskName), Method: "PULL", ETA: time.Unix(0, *t.EtaUsec*1e3), RetryCount: *t.RetryCount, Tag: string(t.Tag), } } return tasks, nil }
// fetch fetches readBufferSize bytes starting at the given offset. On success, // the data is saved as r.buf. func (r *reader) fetch(off int64) error { req := &blobpb.FetchDataRequest{ BlobKey: proto.String(string(r.blobKey)), StartIndex: proto.Int64(off), EndIndex: proto.Int64(off + readBufferSize - 1), // EndIndex is inclusive. } res := &blobpb.FetchDataResponse{} if err := internal.Call(r.c, "blobstore", "FetchData", req, res); err != nil { return err } if len(res.Data) == 0 { return io.EOF } r.buf, r.r, r.off = res.Data, 0, off return nil }
func logf(c *context, level int64, format string, args ...interface{}) { s := fmt.Sprintf(format, args...) s = strings.TrimRight(s, "\n") // Remove any trailing newline characters. c.addLogLine(&logpb.UserAppLogLine{ TimestampUsec: proto.Int64(time.Now().UnixNano() / 1e3), Level: &level, Message: &s, }) log.Print(logLevelName[level] + ": " + s) }
// SetNumInstances sets the number of instances of the given module.version to the // specified value. If either module or version are the empty string it means the // default. func SetNumInstances(c context.Context, module, version string, instances int) error { req := &pb.SetNumInstancesRequest{} if module != "" { req.Module = &module } if version != "" { req.Version = &version } req.Instances = proto.Int64(int64(instances)) res := &pb.SetNumInstancesResponse{} return internal.Call(c, "modules", "SetNumInstances", req, res) }
func interfaceToProto(iv interface{}) (p *pb.Value, errStr string) { val := new(pb.Value) switch v := iv.(type) { case int: val.IntegerValue = proto.Int64(int64(v)) case int32: val.IntegerValue = proto.Int64(int64(v)) case int64: val.IntegerValue = proto.Int64(v) case bool: val.BooleanValue = proto.Bool(v) case string: val.StringValue = proto.String(v) case float32: val.DoubleValue = proto.Float64(float64(v)) case float64: val.DoubleValue = proto.Float64(v) case *Key: if v != nil { val.KeyValue = keyToProto(v) } case time.Time: if v.Before(minTime) || v.After(maxTime) { return nil, fmt.Sprintf("time value out of range") } val.TimestampMicrosecondsValue = proto.Int64(toUnixMicro(v)) case []byte: val.BlobValue = v default: if iv != nil { return nil, fmt.Sprintf("invalid Value type %t", iv) } } // TODO(jbd): Support ListValue and EntityValue. // TODO(jbd): Support types whose underlying type is one of the types above. return val, "" }
// ModifyLease modifies the lease of a task. // Used to request more processing time, or to abandon processing. // leaseTime is in seconds and must not be negative. func ModifyLease(c context.Context, task *Task, queueName string, leaseTime int) error { if queueName == "" { queueName = "default" } req := &pb.TaskQueueModifyTaskLeaseRequest{ QueueName: []byte(queueName), TaskName: []byte(task.Name), EtaUsec: proto.Int64(task.ETA.UnixNano() / 1e3), // Used to verify ownership. LeaseSeconds: proto.Float64(float64(leaseTime)), } res := &pb.TaskQueueModifyTaskLeaseResponse{} if err := internal.Call(c, "taskqueue", "ModifyTaskLease", req, res); err != nil { return err } task.ETA = time.Unix(0, *res.UpdatedEtaUsec*1e3) return nil }
// toRetryParameter converts RetryOptions to pb.TaskQueueRetryParameters. func (opt *RetryOptions) toRetryParameters() *pb.TaskQueueRetryParameters { params := &pb.TaskQueueRetryParameters{} if opt.RetryLimit > 0 { params.RetryLimit = proto.Int32(opt.RetryLimit) } if opt.AgeLimit > 0 { params.AgeLimitSec = proto.Int64(int64(opt.AgeLimit.Seconds())) } if opt.MinBackoff > 0 { params.MinBackoffSec = proto.Float64(opt.MinBackoff.Seconds()) } if opt.MaxBackoff > 0 { params.MaxBackoffSec = proto.Float64(opt.MaxBackoff.Seconds()) } if opt.MaxDoublings > 0 || (opt.MaxDoublings == 0 && opt.ApplyZeroMaxDoublings) { params.MaxDoublings = proto.Int32(opt.MaxDoublings) } return params }
// valueToProto converts a named value to a newly allocated Property. // The returned error string is empty on success. func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) { var ( pv pb.PropertyValue unsupported bool ) switch v.Kind() { case reflect.Invalid: // No-op. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: pv.Int64Value = proto.Int64(v.Int()) case reflect.Bool: pv.BooleanValue = proto.Bool(v.Bool()) case reflect.String: pv.StringValue = proto.String(v.String()) case reflect.Float32, reflect.Float64: pv.DoubleValue = proto.Float64(v.Float()) case reflect.Ptr: if k, ok := v.Interface().(*Key); ok { if k != nil { pv.Referencevalue = keyToReferenceValue(defaultAppID, k) } } else { unsupported = true } case reflect.Struct: switch t := v.Interface().(type) { case time.Time: if t.Before(minTime) || t.After(maxTime) { return nil, "time value out of range" } pv.Int64Value = proto.Int64(toUnixMicro(t)) case appengine.GeoPoint: if !t.Valid() { return nil, "invalid GeoPoint value" } // NOTE: Strangely, latitude maps to X, longitude to Y. pv.Pointvalue = &pb.PropertyValue_PointValue{X: &t.Lat, Y: &t.Lng} default: unsupported = true } case reflect.Slice: if b, ok := v.Interface().([]byte); ok { pv.StringValue = proto.String(string(b)) } else { // nvToProto should already catch slice values. // If we get here, we have a slice of slice values. unsupported = true } default: unsupported = true } if unsupported { return nil, "unsupported datastore value type: " + v.Type().String() } p = &pb.Property{ Name: proto.String(name), Value: &pv, Multiple: proto.Bool(multiple), } if v.IsValid() { switch v.Interface().(type) { case []byte: p.Meaning = pb.Property_BLOB.Enum() case ByteString: p.Meaning = pb.Property_BYTESTRING.Enum() case appengine.BlobKey: p.Meaning = pb.Property_BLOBKEY.Enum() case time.Time: p.Meaning = pb.Property_GD_WHEN.Enum() case appengine.GeoPoint: p.Meaning = pb.Property_GEORSS_POINT.Enum() } } return p, "" }
func propertiesToProto(defaultAppID string, key *Key, props []Property) (*pb.EntityProto, error) { e := &pb.EntityProto{ Key: keyToProto(defaultAppID, key), } if key.parent == nil { e.EntityGroup = &pb.Path{} } else { e.EntityGroup = keyToProto(defaultAppID, key.root()).Path } prevMultiple := make(map[string]bool) for _, p := range props { if pm, ok := prevMultiple[p.Name]; ok { if !pm || !p.Multiple { return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name) } } else { prevMultiple[p.Name] = p.Multiple } x := &pb.Property{ Name: proto.String(p.Name), Value: new(pb.PropertyValue), Multiple: proto.Bool(p.Multiple), } switch v := p.Value.(type) { case int64: x.Value.Int64Value = proto.Int64(v) case bool: x.Value.BooleanValue = proto.Bool(v) case string: x.Value.StringValue = proto.String(v) if p.NoIndex { x.Meaning = pb.Property_TEXT.Enum() } case float64: x.Value.DoubleValue = proto.Float64(v) case *Key: if v != nil { x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v) } case time.Time: if v.Before(minTime) || v.After(maxTime) { return nil, fmt.Errorf("datastore: time value out of range") } x.Value.Int64Value = proto.Int64(toUnixMicro(v)) x.Meaning = pb.Property_GD_WHEN.Enum() case appengine.BlobKey: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BLOBKEY.Enum() case appengine.GeoPoint: if !v.Valid() { return nil, fmt.Errorf("datastore: invalid GeoPoint value") } // NOTE: Strangely, latitude maps to X, longitude to Y. x.Value.Pointvalue = &pb.PropertyValue_PointValue{X: &v.Lat, Y: &v.Lng} x.Meaning = pb.Property_GEORSS_POINT.Enum() case []byte: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BLOB.Enum() if !p.NoIndex { return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name) } case ByteString: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BYTESTRING.Enum() default: if p.Value != nil { return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name) } } if p.NoIndex { e.RawProperty = append(e.RawProperty, x) } else { e.Property = append(e.Property, x) if len(e.Property) > maxIndexedProperties { return nil, errors.New("datastore: too many indexed properties") } } } return e, nil }
func newAddReq(c context.Context, task *Task, queueName string) (*pb.TaskQueueAddRequest, error) { if queueName == "" { queueName = "default" } path := task.Path if path == "" { path = "/_ah/queue/" + queueName } eta := task.ETA if eta.IsZero() { eta = time.Now().Add(task.Delay) } else if task.Delay != 0 { panic("taskqueue: both Delay and ETA are set") } req := &pb.TaskQueueAddRequest{ QueueName: []byte(queueName), TaskName: []byte(task.Name), EtaUsec: proto.Int64(eta.UnixNano() / 1e3), } method := task.method() if method == "PULL" { // Pull-based task req.Body = task.Payload req.Mode = pb.TaskQueueMode_PULL.Enum() if task.Tag != "" { req.Tag = []byte(task.Tag) } } else { // HTTP-based task if v, ok := pb.TaskQueueAddRequest_RequestMethod_value[method]; ok { req.Method = pb.TaskQueueAddRequest_RequestMethod(v).Enum() } else { return nil, fmt.Errorf("taskqueue: bad method %q", method) } req.Url = []byte(path) for k, vs := range task.Header { for _, v := range vs { req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ Key: []byte(k), Value: []byte(v), }) } } if method == "POST" || method == "PUT" { req.Body = task.Payload } // Namespace headers. if _, ok := task.Header[currentNamespace]; !ok { // Fetch the current namespace of this request. ns := internal.NamespaceFromContext(c) req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ Key: []byte(currentNamespace), Value: []byte(ns), }) } if _, ok := task.Header[defaultNamespace]; !ok { // Fetch the X-AppEngine-Default-Namespace header of this request. if ns := getDefaultNamespace(c); ns != "" { req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ Key: []byte(defaultNamespace), Value: []byte(ns), }) } } } if task.RetryOptions != nil { req.RetryParameters = task.RetryOptions.toRetryParameters() } return req, nil }