Exemplo n.º 1
0
func makeRequest(params *Query, appID, versionID string) (*pb.LogReadRequest, error) {
	req := &pb.LogReadRequest{}
	req.AppId = &appID
	if !params.StartTime.IsZero() {
		req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3)
	}
	if !params.EndTime.IsZero() {
		req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3)
	}
	if len(params.Offset) > 0 {
		var offset pb.LogOffset
		if err := proto.Unmarshal(params.Offset, &offset); err != nil {
			return nil, fmt.Errorf("bad Offset: %v", err)
		}
		req.Offset = &offset
	}
	if params.Incomplete {
		req.IncludeIncomplete = &params.Incomplete
	}
	if params.AppLogs {
		req.IncludeAppLogs = &params.AppLogs
	}
	if params.ApplyMinLevel {
		req.MinimumLogLevel = proto.Int32(int32(params.MinLevel))
	}
	if params.Versions == nil {
		// If no versions were specified, default to the default module at
		// the major version being used by this module.
		if i := strings.Index(versionID, "."); i >= 0 {
			versionID = versionID[:i]
		}
		req.VersionId = []string{versionID}
	} else {
		req.ModuleVersion = make([]*pb.LogModuleVersion, 0, len(params.Versions))
		for _, v := range params.Versions {
			var m *string
			if i := strings.Index(v, ":"); i >= 0 {
				m, v = proto.String(v[:i]), v[i+1:]
			}
			req.ModuleVersion = append(req.ModuleVersion, &pb.LogModuleVersion{
				ModuleId:  m,
				VersionId: proto.String(v),
			})
		}
	}
	if params.RequestIDs != nil {
		ids := make([][]byte, len(params.RequestIDs))
		for i, v := range params.RequestIDs {
			ids[i] = []byte(v)
		}
		req.RequestId = ids
	}

	return req, nil
}
Exemplo n.º 2
0
func keyToProto(k *Key) *pb.Key {
	if k == nil {
		return nil
	}

	// TODO(jbd): Eliminate unrequired allocations.
	path := []*pb.Key_PathElement(nil)
	for {
		el := &pb.Key_PathElement{
			Kind: proto.String(k.kind),
		}
		if k.id != 0 {
			el.Id = proto.Int64(k.id)
		}
		if k.name != "" {
			el.Name = proto.String(k.name)
		}
		path = append([]*pb.Key_PathElement{el}, path...)
		if k.parent == nil {
			break
		}
		k = k.parent
	}
	key := &pb.Key{
		PathElement: path,
	}
	if k.namespace != "" {
		key.PartitionId = &pb.PartitionId{
			Namespace: proto.String(k.namespace),
		}
	}
	return key
}
Exemplo n.º 3
0
// AllocateIDs returns a range of n integer IDs with the given kind and parent
// combination. kind cannot be empty; parent may be nil. The IDs in the range
// returned will not be used by the datastore's automatic ID sequence generator
// and may be used with NewKey without conflict.
//
// The range is inclusive at the low end and exclusive at the high end. In
// other words, valid intIDs x satisfy low <= x && x < high.
//
// If no error is returned, low + n == high.
func AllocateIDs(c context.Context, kind string, parent *Key, n int) (low, high int64, err error) {
	if kind == "" {
		return 0, 0, errors.New("datastore: AllocateIDs given an empty kind")
	}
	if n < 0 {
		return 0, 0, fmt.Errorf("datastore: AllocateIDs given a negative count: %d", n)
	}
	if n == 0 {
		return 0, 0, nil
	}
	req := &pb.AllocateIdsRequest{
		ModelKey: keyToProto("", NewIncompleteKey(c, kind, parent)),
		Size:     proto.Int64(int64(n)),
	}
	res := &pb.AllocateIdsResponse{}
	if err := internal.Call(c, "datastore_v3", "AllocateIds", req, res); err != nil {
		return 0, 0, err
	}
	// The protobuf is inclusive at both ends. Idiomatic Go (e.g. slices, for loops)
	// is inclusive at the low end and exclusive at the high end, so we add 1.
	low = res.GetStart()
	high = res.GetEnd() + 1
	if low+int64(n) != high {
		return 0, 0, fmt.Errorf("datastore: internal error: could not allocate %d IDs", n)
	}
	return low, high, nil
}
Exemplo n.º 4
0
func lease(c context.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) {
	if queueName == "" {
		queueName = "default"
	}
	req := &pb.TaskQueueQueryAndOwnTasksRequest{
		QueueName:    []byte(queueName),
		LeaseSeconds: proto.Float64(float64(leaseTime)),
		MaxTasks:     proto.Int64(int64(maxTasks)),
		GroupByTag:   proto.Bool(groupByTag),
		Tag:          tag,
	}
	res := &pb.TaskQueueQueryAndOwnTasksResponse{}
	if err := internal.Call(c, "taskqueue", "QueryAndOwnTasks", req, res); err != nil {
		return nil, err
	}
	tasks := make([]*Task, len(res.Task))
	for i, t := range res.Task {
		tasks[i] = &Task{
			Payload:    t.Body,
			Name:       string(t.TaskName),
			Method:     "PULL",
			ETA:        time.Unix(0, *t.EtaUsec*1e3),
			RetryCount: *t.RetryCount,
			Tag:        string(t.Tag),
		}
	}
	return tasks, nil
}
Exemplo n.º 5
0
// fetch fetches readBufferSize bytes starting at the given offset. On success,
// the data is saved as r.buf.
func (r *reader) fetch(off int64) error {
	req := &blobpb.FetchDataRequest{
		BlobKey:    proto.String(string(r.blobKey)),
		StartIndex: proto.Int64(off),
		EndIndex:   proto.Int64(off + readBufferSize - 1), // EndIndex is inclusive.
	}
	res := &blobpb.FetchDataResponse{}
	if err := internal.Call(r.c, "blobstore", "FetchData", req, res); err != nil {
		return err
	}
	if len(res.Data) == 0 {
		return io.EOF
	}
	r.buf, r.r, r.off = res.Data, 0, off
	return nil
}
Exemplo n.º 6
0
func logf(c *context, level int64, format string, args ...interface{}) {
	s := fmt.Sprintf(format, args...)
	s = strings.TrimRight(s, "\n") // Remove any trailing newline characters.
	c.addLogLine(&logpb.UserAppLogLine{
		TimestampUsec: proto.Int64(time.Now().UnixNano() / 1e3),
		Level:         &level,
		Message:       &s,
	})
	log.Print(logLevelName[level] + ": " + s)
}
Exemplo n.º 7
0
// SetNumInstances sets the number of instances of the given module.version to the
// specified value. If either module or version are the empty string it means the
// default.
func SetNumInstances(c context.Context, module, version string, instances int) error {
	req := &pb.SetNumInstancesRequest{}
	if module != "" {
		req.Module = &module
	}
	if version != "" {
		req.Version = &version
	}
	req.Instances = proto.Int64(int64(instances))
	res := &pb.SetNumInstancesResponse{}
	return internal.Call(c, "modules", "SetNumInstances", req, res)
}
Exemplo n.º 8
0
func interfaceToProto(iv interface{}) (p *pb.Value, errStr string) {
	val := new(pb.Value)
	switch v := iv.(type) {
	case int:
		val.IntegerValue = proto.Int64(int64(v))
	case int32:
		val.IntegerValue = proto.Int64(int64(v))
	case int64:
		val.IntegerValue = proto.Int64(v)
	case bool:
		val.BooleanValue = proto.Bool(v)
	case string:
		val.StringValue = proto.String(v)
	case float32:
		val.DoubleValue = proto.Float64(float64(v))
	case float64:
		val.DoubleValue = proto.Float64(v)
	case *Key:
		if v != nil {
			val.KeyValue = keyToProto(v)
		}
	case time.Time:
		if v.Before(minTime) || v.After(maxTime) {
			return nil, fmt.Sprintf("time value out of range")
		}
		val.TimestampMicrosecondsValue = proto.Int64(toUnixMicro(v))
	case []byte:
		val.BlobValue = v
	default:
		if iv != nil {
			return nil, fmt.Sprintf("invalid Value type %t", iv)
		}
	}
	// TODO(jbd): Support ListValue and EntityValue.
	// TODO(jbd): Support types whose underlying type is one of the types above.
	return val, ""
}
Exemplo n.º 9
0
// ModifyLease modifies the lease of a task.
// Used to request more processing time, or to abandon processing.
// leaseTime is in seconds and must not be negative.
func ModifyLease(c context.Context, task *Task, queueName string, leaseTime int) error {
	if queueName == "" {
		queueName = "default"
	}
	req := &pb.TaskQueueModifyTaskLeaseRequest{
		QueueName:    []byte(queueName),
		TaskName:     []byte(task.Name),
		EtaUsec:      proto.Int64(task.ETA.UnixNano() / 1e3), // Used to verify ownership.
		LeaseSeconds: proto.Float64(float64(leaseTime)),
	}
	res := &pb.TaskQueueModifyTaskLeaseResponse{}
	if err := internal.Call(c, "taskqueue", "ModifyTaskLease", req, res); err != nil {
		return err
	}
	task.ETA = time.Unix(0, *res.UpdatedEtaUsec*1e3)
	return nil
}
Exemplo n.º 10
0
// toRetryParameter converts RetryOptions to pb.TaskQueueRetryParameters.
func (opt *RetryOptions) toRetryParameters() *pb.TaskQueueRetryParameters {
	params := &pb.TaskQueueRetryParameters{}
	if opt.RetryLimit > 0 {
		params.RetryLimit = proto.Int32(opt.RetryLimit)
	}
	if opt.AgeLimit > 0 {
		params.AgeLimitSec = proto.Int64(int64(opt.AgeLimit.Seconds()))
	}
	if opt.MinBackoff > 0 {
		params.MinBackoffSec = proto.Float64(opt.MinBackoff.Seconds())
	}
	if opt.MaxBackoff > 0 {
		params.MaxBackoffSec = proto.Float64(opt.MaxBackoff.Seconds())
	}
	if opt.MaxDoublings > 0 || (opt.MaxDoublings == 0 && opt.ApplyZeroMaxDoublings) {
		params.MaxDoublings = proto.Int32(opt.MaxDoublings)
	}
	return params
}
Exemplo n.º 11
0
// valueToProto converts a named value to a newly allocated Property.
// The returned error string is empty on success.
func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) {
	var (
		pv          pb.PropertyValue
		unsupported bool
	)
	switch v.Kind() {
	case reflect.Invalid:
		// No-op.
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		pv.Int64Value = proto.Int64(v.Int())
	case reflect.Bool:
		pv.BooleanValue = proto.Bool(v.Bool())
	case reflect.String:
		pv.StringValue = proto.String(v.String())
	case reflect.Float32, reflect.Float64:
		pv.DoubleValue = proto.Float64(v.Float())
	case reflect.Ptr:
		if k, ok := v.Interface().(*Key); ok {
			if k != nil {
				pv.Referencevalue = keyToReferenceValue(defaultAppID, k)
			}
		} else {
			unsupported = true
		}
	case reflect.Struct:
		switch t := v.Interface().(type) {
		case time.Time:
			if t.Before(minTime) || t.After(maxTime) {
				return nil, "time value out of range"
			}
			pv.Int64Value = proto.Int64(toUnixMicro(t))
		case appengine.GeoPoint:
			if !t.Valid() {
				return nil, "invalid GeoPoint value"
			}
			// NOTE: Strangely, latitude maps to X, longitude to Y.
			pv.Pointvalue = &pb.PropertyValue_PointValue{X: &t.Lat, Y: &t.Lng}
		default:
			unsupported = true
		}
	case reflect.Slice:
		if b, ok := v.Interface().([]byte); ok {
			pv.StringValue = proto.String(string(b))
		} else {
			// nvToProto should already catch slice values.
			// If we get here, we have a slice of slice values.
			unsupported = true
		}
	default:
		unsupported = true
	}
	if unsupported {
		return nil, "unsupported datastore value type: " + v.Type().String()
	}
	p = &pb.Property{
		Name:     proto.String(name),
		Value:    &pv,
		Multiple: proto.Bool(multiple),
	}
	if v.IsValid() {
		switch v.Interface().(type) {
		case []byte:
			p.Meaning = pb.Property_BLOB.Enum()
		case ByteString:
			p.Meaning = pb.Property_BYTESTRING.Enum()
		case appengine.BlobKey:
			p.Meaning = pb.Property_BLOBKEY.Enum()
		case time.Time:
			p.Meaning = pb.Property_GD_WHEN.Enum()
		case appengine.GeoPoint:
			p.Meaning = pb.Property_GEORSS_POINT.Enum()
		}
	}
	return p, ""
}
Exemplo n.º 12
0
func propertiesToProto(defaultAppID string, key *Key, props []Property) (*pb.EntityProto, error) {
	e := &pb.EntityProto{
		Key: keyToProto(defaultAppID, key),
	}
	if key.parent == nil {
		e.EntityGroup = &pb.Path{}
	} else {
		e.EntityGroup = keyToProto(defaultAppID, key.root()).Path
	}
	prevMultiple := make(map[string]bool)

	for _, p := range props {
		if pm, ok := prevMultiple[p.Name]; ok {
			if !pm || !p.Multiple {
				return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name)
			}
		} else {
			prevMultiple[p.Name] = p.Multiple
		}

		x := &pb.Property{
			Name:     proto.String(p.Name),
			Value:    new(pb.PropertyValue),
			Multiple: proto.Bool(p.Multiple),
		}
		switch v := p.Value.(type) {
		case int64:
			x.Value.Int64Value = proto.Int64(v)
		case bool:
			x.Value.BooleanValue = proto.Bool(v)
		case string:
			x.Value.StringValue = proto.String(v)
			if p.NoIndex {
				x.Meaning = pb.Property_TEXT.Enum()
			}
		case float64:
			x.Value.DoubleValue = proto.Float64(v)
		case *Key:
			if v != nil {
				x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v)
			}
		case time.Time:
			if v.Before(minTime) || v.After(maxTime) {
				return nil, fmt.Errorf("datastore: time value out of range")
			}
			x.Value.Int64Value = proto.Int64(toUnixMicro(v))
			x.Meaning = pb.Property_GD_WHEN.Enum()
		case appengine.BlobKey:
			x.Value.StringValue = proto.String(string(v))
			x.Meaning = pb.Property_BLOBKEY.Enum()
		case appengine.GeoPoint:
			if !v.Valid() {
				return nil, fmt.Errorf("datastore: invalid GeoPoint value")
			}
			// NOTE: Strangely, latitude maps to X, longitude to Y.
			x.Value.Pointvalue = &pb.PropertyValue_PointValue{X: &v.Lat, Y: &v.Lng}
			x.Meaning = pb.Property_GEORSS_POINT.Enum()
		case []byte:
			x.Value.StringValue = proto.String(string(v))
			x.Meaning = pb.Property_BLOB.Enum()
			if !p.NoIndex {
				return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name)
			}
		case ByteString:
			x.Value.StringValue = proto.String(string(v))
			x.Meaning = pb.Property_BYTESTRING.Enum()
		default:
			if p.Value != nil {
				return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name)
			}
		}

		if p.NoIndex {
			e.RawProperty = append(e.RawProperty, x)
		} else {
			e.Property = append(e.Property, x)
			if len(e.Property) > maxIndexedProperties {
				return nil, errors.New("datastore: too many indexed properties")
			}
		}
	}
	return e, nil
}
Exemplo n.º 13
0
func newAddReq(c context.Context, task *Task, queueName string) (*pb.TaskQueueAddRequest, error) {
	if queueName == "" {
		queueName = "default"
	}
	path := task.Path
	if path == "" {
		path = "/_ah/queue/" + queueName
	}
	eta := task.ETA
	if eta.IsZero() {
		eta = time.Now().Add(task.Delay)
	} else if task.Delay != 0 {
		panic("taskqueue: both Delay and ETA are set")
	}
	req := &pb.TaskQueueAddRequest{
		QueueName: []byte(queueName),
		TaskName:  []byte(task.Name),
		EtaUsec:   proto.Int64(eta.UnixNano() / 1e3),
	}
	method := task.method()
	if method == "PULL" {
		// Pull-based task
		req.Body = task.Payload
		req.Mode = pb.TaskQueueMode_PULL.Enum()
		if task.Tag != "" {
			req.Tag = []byte(task.Tag)
		}
	} else {
		// HTTP-based task
		if v, ok := pb.TaskQueueAddRequest_RequestMethod_value[method]; ok {
			req.Method = pb.TaskQueueAddRequest_RequestMethod(v).Enum()
		} else {
			return nil, fmt.Errorf("taskqueue: bad method %q", method)
		}
		req.Url = []byte(path)
		for k, vs := range task.Header {
			for _, v := range vs {
				req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{
					Key:   []byte(k),
					Value: []byte(v),
				})
			}
		}
		if method == "POST" || method == "PUT" {
			req.Body = task.Payload
		}

		// Namespace headers.
		if _, ok := task.Header[currentNamespace]; !ok {
			// Fetch the current namespace of this request.
			ns := internal.NamespaceFromContext(c)
			req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{
				Key:   []byte(currentNamespace),
				Value: []byte(ns),
			})
		}
		if _, ok := task.Header[defaultNamespace]; !ok {
			// Fetch the X-AppEngine-Default-Namespace header of this request.
			if ns := getDefaultNamespace(c); ns != "" {
				req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{
					Key:   []byte(defaultNamespace),
					Value: []byte(ns),
				})
			}
		}
	}

	if task.RetryOptions != nil {
		req.RetryParameters = task.RetryOptions.toRetryParameters()
	}

	return req, nil
}