func (cn *Conn) Write(b []byte) (n int, err error) { const lim = 1 << 20 // max per chunk for n < len(b) { chunk := b[n:] if len(chunk) > lim { chunk = chunk[:lim] } req := &pb.SendRequest{ SocketDescriptor: &cn.desc, Data: chunk, StreamOffset: &cn.offset, } res := &pb.SendReply{} if !cn.writeDeadline.IsZero() { req.TimeoutSeconds = proto.Float64(cn.writeDeadline.Sub(time.Now()).Seconds()) } if err = cn.c.Call("remote_socket", "Send", req, res, nil); err != nil { // assume zero bytes were sent in this RPC break } n += int(res.GetDataSent()) } cn.offset += int64(n) return }
func (cn *Conn) Read(b []byte) (n int, err error) { const maxRead = 1 << 20 if len(b) > maxRead { b = b[:maxRead] } req := &pb.ReceiveRequest{ SocketDescriptor: &cn.desc, DataSize: proto.Int32(int32(len(b))), } res := &pb.ReceiveReply{} if !cn.readDeadline.IsZero() { req.TimeoutSeconds = proto.Float64(cn.readDeadline.Sub(time.Now()).Seconds()) } if err := cn.c.Call("remote_socket", "Receive", req, res, nil); err != nil { return 0, err } if len(res.Data) == 0 { return 0, io.EOF } if len(res.Data) > len(b) { return 0, fmt.Errorf("socket: internal error: read too much data: %d > %d", len(res.Data), len(b)) } return copy(b, res.Data), nil }
func lease(c appengine.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) { if queueName == "" { queueName = "default" } req := &pb.TaskQueueQueryAndOwnTasksRequest{ QueueName: []byte(queueName), LeaseSeconds: proto.Float64(float64(leaseTime)), MaxTasks: proto.Int64(int64(maxTasks)), GroupByTag: proto.Bool(groupByTag), Tag: tag, } res := &pb.TaskQueueQueryAndOwnTasksResponse{} callOpts := &appengine_internal.CallOptions{ Timeout: 10 * time.Second, } if err := c.Call("taskqueue", "QueryAndOwnTasks", req, res, callOpts); err != nil { return nil, err } tasks := make([]*Task, len(res.Task)) for i, t := range res.Task { tasks[i] = &Task{ Payload: t.Body, Name: string(t.TaskName), Method: "PULL", ETA: time.Unix(0, *t.EtaUsec*1e3), RetryCount: *t.RetryCount, Tag: string(t.Tag), } } return tasks, nil }
// toRetryParameter converts RetryOptions to pb.TaskQueueRetryParameters. func (opt *RetryOptions) toRetryParameters() *pb.TaskQueueRetryParameters { params := &pb.TaskQueueRetryParameters{} if opt.RetryLimit > 0 { params.RetryLimit = proto.Int32(opt.RetryLimit) } if opt.AgeLimit > 0 { params.AgeLimitSec = proto.Int64(int64(opt.AgeLimit.Seconds())) } if opt.MinBackoff > 0 { params.MinBackoffSec = proto.Float64(opt.MinBackoff.Seconds()) } if opt.MaxBackoff > 0 { params.MaxBackoffSec = proto.Float64(opt.MaxBackoff.Seconds()) } if opt.MaxDoublings > 0 || (opt.MaxDoublings == 0 && opt.ApplyZeroMaxDoublings) { params.MaxDoublings = proto.Int32(opt.MaxDoublings) } return params }
// ModifyLease modifies the lease of a task. // Used to request more processing time, or to abandon processing. // leaseTime is in seconds and must not be negative. func ModifyLease(c appengine.Context, task *Task, queueName string, leaseTime int) error { if queueName == "" { queueName = "default" } req := &pb.TaskQueueModifyTaskLeaseRequest{ QueueName: []byte(queueName), TaskName: []byte(task.Name), EtaUsec: proto.Int64(task.ETA.UnixNano() / 1e3), // Used to verify ownership. LeaseSeconds: proto.Float64(float64(leaseTime)), } res := &pb.TaskQueueModifyTaskLeaseResponse{} if err := c.Call("taskqueue", "ModifyTaskLease", req, res, nil); err != nil { return err } task.ETA = time.Unix(0, *res.UpdatedEtaUsec*1e3) return nil }
// RoundTrip issues a single HTTP request and returns its response. Per the // http.RoundTripper interface, RoundTrip only returns an error if there // was an unsupported request or the URL Fetch proxy fails. // Note that HTTP response codes such as 5xx, 403, 404, etc are not // errors as far as the transport is concerned and will be returned // with err set to nil. func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) { methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method] if !ok { return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method) } method := pb.URLFetchRequest_RequestMethod(methNum) freq := &pb.URLFetchRequest{ Method: &method, Url: proto.String(urlString(req.URL)), FollowRedirects: proto.Bool(false), // http.Client's responsibility MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate), } opts := &appengine_internal.CallOptions{} if t.Deadline != 0 { freq.Deadline = proto.Float64(t.Deadline.Seconds()) opts.Timeout = t.Deadline } for k, vals := range req.Header { for _, val := range vals { freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{ Key: proto.String(k), Value: proto.String(val), }) } } if methodAcceptsRequestBody[req.Method] && req.Body != nil { freq.Payload, err = ioutil.ReadAll(req.Body) if err != nil { return nil, err } } fres := &pb.URLFetchResponse{} if err := t.Context.Call("urlfetch", "Fetch", freq, fres, opts); err != nil { return nil, err } res = &http.Response{} res.StatusCode = int(*fres.StatusCode) res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode)) res.Header = make(http.Header) res.Request = req // Faked: res.ProtoMajor = 1 res.ProtoMinor = 1 res.Proto = "HTTP/1.1" res.Close = true for _, h := range fres.Header { hkey := http.CanonicalHeaderKey(*h.Key) hval := *h.Value if hkey == "Content-Length" { // Will get filled in below for all but HEAD requests. if req.Method == "HEAD" { res.ContentLength, _ = strconv.ParseInt(hval, 10, 64) } continue } res.Header.Add(hkey, hval) } if req.Method != "HEAD" { res.ContentLength = int64(len(fres.Content)) } truncated := fres.GetContentWasTruncated() res.Body = &bodyReader{content: fres.Content, truncated: truncated} return }
// valueToProto converts a named value to a newly allocated Property. // The returned error string is empty on success. func valueToProto(defaultAppID, name string, v reflect.Value, multiple bool) (p *pb.Property, errStr string) { var ( pv pb.PropertyValue unsupported bool ) switch v.Kind() { case reflect.Invalid: // No-op. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: pv.Int64Value = proto.Int64(v.Int()) case reflect.Bool: pv.BooleanValue = proto.Bool(v.Bool()) case reflect.String: pv.StringValue = proto.String(v.String()) case reflect.Float32, reflect.Float64: pv.DoubleValue = proto.Float64(v.Float()) case reflect.Ptr: if k, ok := v.Interface().(*Key); ok { if k != nil { pv.Referencevalue = keyToReferenceValue(defaultAppID, k) } } else { unsupported = true } case reflect.Struct: if t, ok := v.Interface().(time.Time); ok { if t.Before(minTime) || t.After(maxTime) { return nil, "time value out of range" } pv.Int64Value = proto.Int64(toUnixMicro(t)) } else { unsupported = true } case reflect.Slice: if b, ok := v.Interface().([]byte); ok { pv.StringValue = proto.String(string(b)) } else { // nvToProto should already catch slice values. // If we get here, we have a slice of slice values. unsupported = true } default: unsupported = true } if unsupported { return nil, "unsupported datastore value type: " + v.Type().String() } p = &pb.Property{ Name: proto.String(name), Value: &pv, Multiple: proto.Bool(multiple), } if v.IsValid() { switch v.Interface().(type) { case []byte: p.Meaning = pb.Property_BLOB.Enum() case appengine.BlobKey: p.Meaning = pb.Property_BLOBKEY.Enum() case time.Time: p.Meaning = pb.Property_GD_WHEN.Enum() } } return p, "" }
func propertiesToProto(defaultAppID string, key *Key, src <-chan Property) (*pb.EntityProto, error) { defer func() { for _ = range src { // Drain the src channel, if we exit early. } }() e := &pb.EntityProto{ Key: keyToProto(defaultAppID, key), } if key.parent == nil { e.EntityGroup = &pb.Path{} } else { e.EntityGroup = keyToProto(defaultAppID, key.root()).Path } prevMultiple := make(map[string]bool) for p := range src { if pm, ok := prevMultiple[p.Name]; ok { if !pm || !p.Multiple { return nil, fmt.Errorf("datastore: multiple Properties with Name %q, but Multiple is false", p.Name) } } else { prevMultiple[p.Name] = p.Multiple } x := &pb.Property{ Name: proto.String(p.Name), Value: new(pb.PropertyValue), Multiple: proto.Bool(p.Multiple), } switch v := p.Value.(type) { case int64: x.Value.Int64Value = proto.Int64(v) case bool: x.Value.BooleanValue = proto.Bool(v) case string: x.Value.StringValue = proto.String(v) if p.NoIndex { x.Meaning = pb.Property_TEXT.Enum() } case float64: x.Value.DoubleValue = proto.Float64(v) case *Key: if v != nil { x.Value.Referencevalue = keyToReferenceValue(defaultAppID, v) } case time.Time: if v.Before(minTime) || v.After(maxTime) { return nil, fmt.Errorf("datastore: time value out of range") } x.Value.Int64Value = proto.Int64(toUnixMicro(v)) x.Meaning = pb.Property_GD_WHEN.Enum() case appengine.BlobKey: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BLOBKEY.Enum() case []byte: x.Value.StringValue = proto.String(string(v)) x.Meaning = pb.Property_BLOB.Enum() if !p.NoIndex { return nil, fmt.Errorf("datastore: cannot index a []byte valued Property with Name %q", p.Name) } default: if p.Value != nil { return nil, fmt.Errorf("datastore: invalid Value type for a Property with Name %q", p.Name) } } if p.NoIndex { e.RawProperty = append(e.RawProperty, x) } else { e.Property = append(e.Property, x) if len(e.Property) > maxIndexedProperties { return nil, errors.New("datastore: too many indexed properties") } } } return e, nil }