func (cn *Conn) Write(b []byte) (n int, err error) {
const lim = 1 << 20 // max per chunk
for n < len(b) {
chunk := b[n:]
if len(chunk) > lim {
chunk = chunk[:lim]
}
req := &pb.SendRequest{
SocketDescriptor: &cn.desc,
Data: chunk,
StreamOffset: &cn.offset,
}
res := &pb.SendReply{}
if !cn.writeDeadline.IsZero() {
req.TimeoutSeconds = proto.Float64(cn.writeDeadline.Sub(time.Now()).Seconds())
}
if err = cn.c.Call("remote_socket", "Send", req, res, nil); err != nil {
// assume zero bytes were sent in this RPC
break
}
n += int(res.GetDataSent())
}
cn.offset += int64(n)
return
}
func (cn *Conn) Read(b []byte) (n int, err error) {
const maxRead = 1 << 20
if len(b) > maxRead {
b = b[:maxRead]
}
req := &pb.ReceiveRequest{
SocketDescriptor: &cn.desc,
DataSize: proto.Int32(int32(len(b))),
}
res := &pb.ReceiveReply{}
if !cn.readDeadline.IsZero() {
req.TimeoutSeconds = proto.Float64(cn.readDeadline.Sub(time.Now()).Seconds())
}
if err := cn.c.Call("remote_socket", "Receive", req, res, nil); err != nil {
return 0, err
}
if len(res.Data) == 0 {
return 0, io.EOF
}
if len(res.Data) > len(b) {
return 0, fmt.Errorf("socket: internal error: read too much data: %d > %d", len(res.Data), len(b))
}
return copy(b, res.Data), nil
}
func lease(c appengine.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) {
if queueName == "" {
queueName = "default"
}
req := &pb.TaskQueueQueryAndOwnTasksRequest{
QueueName: []byte(queueName),
LeaseSeconds: proto.Float64(float64(leaseTime)),
MaxTasks: proto.Int64(int64(maxTasks)),
GroupByTag: proto.Bool(groupByTag),
Tag: tag,
}
res := &pb.TaskQueueQueryAndOwnTasksResponse{}
callOpts := &appengine_internal.CallOptions{
Timeout: 10 * time.Second,
}
if err := c.Call("taskqueue", "QueryAndOwnTasks", req, res, callOpts); err != nil {
return nil, err
}
tasks := make([]*Task, len(res.Task))
for i, t := range res.Task {
tasks[i] = &Task{
Payload: t.Body,
Name: string(t.TaskName),
Method: "PULL",
ETA: time.Unix(0, *t.EtaUsec*1e3),
RetryCount: *t.RetryCount,
Tag: string(t.Tag),
}
}
return tasks, nil
}
// toRetryParameter converts RetryOptions to pb.TaskQueueRetryParameters.
func (opt *RetryOptions) toRetryParameters() *pb.TaskQueueRetryParameters {
params := &pb.TaskQueueRetryParameters{}
if opt.RetryLimit > 0 {
params.RetryLimit = proto.Int32(opt.RetryLimit)
}
if opt.AgeLimit > 0 {
params.AgeLimitSec = proto.Int64(int64(opt.AgeLimit.Seconds()))
}
if opt.MinBackoff > 0 {
params.MinBackoffSec = proto.Float64(opt.MinBackoff.Seconds())
}
if opt.MaxBackoff > 0 {
params.MaxBackoffSec = proto.Float64(opt.MaxBackoff.Seconds())
}
if opt.MaxDoublings > 0 || (opt.MaxDoublings == 0 && opt.ApplyZeroMaxDoublings) {
params.MaxDoublings = proto.Int32(opt.MaxDoublings)
}
return params
}
// ModifyLease modifies the lease of a task.
// Used to request more processing time, or to abandon processing.
// leaseTime is in seconds and must not be negative.
func ModifyLease(c appengine.Context, task *Task, queueName string, leaseTime int) error {
if queueName == "" {
queueName = "default"
}
req := &pb.TaskQueueModifyTaskLeaseRequest{
QueueName: []byte(queueName),
TaskName: []byte(task.Name),
EtaUsec: proto.Int64(task.ETA.UnixNano() / 1e3), // Used to verify ownership.
LeaseSeconds: proto.Float64(float64(leaseTime)),
}
res := &pb.TaskQueueModifyTaskLeaseResponse{}
if err := c.Call("taskqueue", "ModifyTaskLease", req, res, nil); err != nil {
return err
}
task.ETA = time.Unix(0, *res.UpdatedEtaUsec*1e3)
return nil
}
// RoundTrip issues a single HTTP request and returns its response. Per the
// http.RoundTripper interface, RoundTrip only returns an error if there
// was an unsupported request or the URL Fetch proxy fails.
// Note that HTTP response codes such as 5xx, 403, 404, etc are not
// errors as far as the transport is concerned and will be returned
// with err set to nil.
func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) {
methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method]
if !ok {
return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method)
}
method := pb.URLFetchRequest_RequestMethod(methNum)
freq := &pb.URLFetchRequest{
Method: &method,
Url: proto.String(urlString(req.URL)),
FollowRedirects: proto.Bool(false), // http.Client's responsibility
MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate),
}
opts := &appengine_internal.CallOptions{}
if t.Deadline != 0 {
freq.Deadline = proto.Float64(t.Deadline.Seconds())
opts.Timeout = t.Deadline
}
for k, vals := range req.Header {
for _, val := range vals {
freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{
Key: proto.String(k),
Value: proto.String(val),
})
}
}
if methodAcceptsRequestBody[req.Method] && req.Body != nil {
freq.Payload, err = ioutil.ReadAll(req.Body)
if err != nil {
return nil, err
}
}
fres := &pb.URLFetchResponse{}
if err := t.Context.Call("urlfetch", "Fetch", freq, fres, opts); err != nil {
return nil, err
}
res = &http.Response{}
res.StatusCode = int(*fres.StatusCode)
res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode))
res.Header = make(http.Header)
res.Request = req
// Faked:
res.ProtoMajor = 1
res.ProtoMinor = 1
res.Proto = "HTTP/1.1"
res.Close = true
for _, h := range fres.Header {
hkey := http.CanonicalHeaderKey(*h.Key)
hval := *h.Value
if hkey == "Content-Length" {
// Will get filled in below for all but HEAD requests.
if req.Method == "HEAD" {
res.ContentLength, _ = strconv.ParseInt(hval, 10, 64)
}
continue
}
res.Header.Add(hkey, hval)
}
if req.Method != "HEAD" {
res.ContentLength = int64(len(fres.Content))
}
truncated := fres.GetContentWasTruncated()
res.Body = &bodyReader{content: fres.Content, truncated: truncated}
return
}
// valueToProto converts a named value to a newly allocated Property.
// The returned error string is empty on success.
func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) {
var (
pv pb.PropertyValue
unsupported bool
)
switch v.Kind() {
case reflect.Invalid:
// No-op.
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
pv.Int64Value = proto.Int64(v.Int())
case reflect.Bool:
pv.BooleanValue = proto.Bool(v.Bool())
case reflect.String:
pv.StringValue = proto.String(v.String())
case reflect.Float32, reflect.Float64:
pv.DoubleValue = proto.Float64(v.Float())
case reflect.Ptr:
if k, ok := v.Interface().(*Key); ok {
if k != nil {
pv.Referencevalue = keyToReferenceValue(defaultAppID, k)
}
} else {
unsupported = true
}
case reflect.Struct:
if t, ok := v.Interface().(time.Time); ok {
if t.Before(minTime) || t.After(maxTime) {
return nil, "time value out of range"
}
pv.Int64Value = proto.Int64(toUnixMicro(t))
} else {
unsupported = true
}
case reflect.Slice:
if b, ok := v.Interface().([]byte); ok {
pv.StringValue = proto.String(string(b))
} else {
// nvToProto should already catch slice values.
// If we get here, we have a slice of slice values.
unsupported = true
}
default:
unsupported = true
}
if unsupported {
return nil, "unsupported datastore value type: " + v.Type().String()
}
p = &pb.Property{
Name: proto.String(name),
Value: &pv,
Multiple: proto.Bool(multiple),
}
if v.IsValid() {
switch v.Interface().(type) {
case []byte:
p.Meaning = pb.Property_BLOB.Enum()
case appengine.BlobKey:
p.Meaning = pb.Property_BLOBKEY.Enum()
case time.Time:
p.Meaning = pb.Property_GD_WHEN.Enum()
}
}
return p, ""
}
func propertiesToProto(defaultAppID string, key *Key, src <-chan Property) (*pb.EntityProto, error) {
defer func() {
for _ = range src {
// Drain the src channel, if we exit early.
}
}()
e := &pb.EntityProto{
Key: keyToProto(defaultAppID, key),
}
if key.parent == nil {
e.EntityGroup = &pb.Path{}
} else {
e.EntityGroup = keyToProto(defaultAppID, key.root()).Path
}
prevMultiple := make(map[string]bool)
for p := range src {
if pm, ok := prevMultiple[p.Name]; ok {
if !pm || !p.Multiple {
return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name)
}
} else {
prevMultiple[p.Name] = p.Multiple
}
x := &pb.Property{
Name: proto.String(p.Name),
Value: new(pb.PropertyValue),
Multiple: proto.Bool(p.Multiple),
}
switch v := p.Value.(type) {
case int64:
x.Value.Int64Value = proto.Int64(v)
case bool:
x.Value.BooleanValue = proto.Bool(v)
case string:
x.Value.StringValue = proto.String(v)
if p.NoIndex {
x.Meaning = pb.Property_TEXT.Enum()
}
case float64:
x.Value.DoubleValue = proto.Float64(v)
case *Key:
if v != nil {
x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v)
}
case time.Time:
if v.Before(minTime) || v.After(maxTime) {
return nil, fmt.Errorf("datastore: time value out of range")
}
x.Value.Int64Value = proto.Int64(toUnixMicro(v))
x.Meaning = pb.Property_GD_WHEN.Enum()
case appengine.BlobKey:
x.Value.StringValue = proto.String(string(v))
x.Meaning = pb.Property_BLOBKEY.Enum()
case []byte:
x.Value.StringValue = proto.String(string(v))
x.Meaning = pb.Property_BLOB.Enum()
if !p.NoIndex {
return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name)
}
default:
if p.Value != nil {
return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name)
}
}
if p.NoIndex {
e.RawProperty = append(e.RawProperty, x)
} else {
e.Property = append(e.Property, x)
if len(e.Property) > maxIndexedProperties {
return nil, errors.New("datastore: too many indexed properties")
}
}
}
return e, nil
}