func (c *remoteContext) call(ctx context.Context, service, method string, in, out proto.Message) error { req, err := proto.Marshal(in) if err != nil { return fmt.Errorf("error marshalling request: %v", err) } remReq := &pb.Request{ ServiceName: proto.String(service), Method: proto.String(method), Request: req, // NOTE(djd): RequestId is unused in the server. } req, err = proto.Marshal(remReq) if err != nil { return fmt.Errorf("proto.Marshal: %v", err) } // TODO(djd): Respect ctx.Deadline()? resp, err := c.client.Post(c.url, "application/octet-stream", bytes.NewReader(req)) if err != nil { return fmt.Errorf("error sending request: %v", err) } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad response %d; body: %q", resp.StatusCode, body) } if err != nil { return fmt.Errorf("failed reading response: %v", err) } remResp := &pb.Response{} if err := proto.Unmarshal(body, remResp); err != nil { return fmt.Errorf("error unmarshalling response: %v", err) } if ae := remResp.GetApplicationError(); ae != nil { return &internal.APIError{ Code: ae.GetCode(), Detail: ae.GetDetail(), Service: service, } } if remResp.Response == nil { return fmt.Errorf("unexpected response: %s", proto.MarshalTextString(remResp)) } return proto.Unmarshal(remResp.Response, out) }
// flushLog attempts to flush any pending logs to the appserver. // It should not be called concurrently. func (c *context) flushLog(force bool) (flushed bool) { c.pendingLogs.Lock() // Grab up to 30 MB. We can get away with up to 32 MB, but let's be cautious. n, rem := 0, 30<<20 for ; n < len(c.pendingLogs.lines); n++ { ll := c.pendingLogs.lines[n] // Each log line will require about 3 bytes of overhead. nb := proto.Size(ll) + 3 if nb > rem { break } rem -= nb } lines := c.pendingLogs.lines[:n] c.pendingLogs.lines = c.pendingLogs.lines[n:] c.pendingLogs.Unlock() if len(lines) == 0 && !force { // Nothing to flush. return false } rescueLogs := false defer func() { if rescueLogs { c.pendingLogs.Lock() c.pendingLogs.lines = append(lines, c.pendingLogs.lines...) c.pendingLogs.Unlock() } }() buf, err := proto.Marshal(&logpb.UserAppLogGroup{ LogLine: lines, }) if err != nil { log.Printf("internal.flushLog: marshaling UserAppLogGroup: %v", err) rescueLogs = true return false } req := &logpb.FlushRequest{ Logs: buf, } res := &basepb.VoidProto{} c.pendingLogs.Lock() c.pendingLogs.flushes++ c.pendingLogs.Unlock() if err := Call(toContext(c), "logservice", "Flush", req, res); err != nil { log.Printf("internal.flushLog: Flush RPC: %v", err) rescueLogs = true return false } return true }
// Encode returns an opaque representation of the key // suitable for use in HTML and URLs. // This is compatible with the Python and Java runtimes. func (k *Key) Encode() string { ref := keyToProto("", k) b, err := proto.Marshal(ref) if err != nil { panic(err) } // Trailing padding is stripped. return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") }
// String returns a base-64 string representation of a cursor. func (c Cursor) String() string { if c.cc == nil { return "" } b, err := proto.Marshal(c.cc) if err != nil { // The only way to construct a Cursor with a non-nil cc field is to // unmarshal from the byte representation. We panic if the unmarshal // succeeds but the marshaling of the unchanged protobuf value fails. panic(fmt.Sprintf("datastore: internal error: malformed cursor: %v", err)) } return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") }
func (c *ProtoClient) Call(ctx context.Context, method string, req, resp proto.Message) error { payload, err := proto.Marshal(req) if err != nil { return err } httpReq, err := http.NewRequest("POST", c.endpoint+method, bytes.NewReader(payload)) if err != nil { return err } httpReq.Header.Set("Content-Type", "application/x-protobuf") if ua := c.userAgent; ua != "" { httpReq.Header.Set("User-Agent", ua) } errc := make(chan error, 1) cancel := makeReqCancel(httpReq) go func() { r, err := c.client.Do(httpReq) if err != nil { errc <- err return } defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) if r.StatusCode != http.StatusOK { err = &ErrHTTP{ StatusCode: r.StatusCode, Body: body, err: err, } } if err != nil { errc <- err return } errc <- proto.Unmarshal(body, resp) }() select { case <-ctx.Done(): cancel(c.client.Transport) // Cancel the HTTP request. return ctx.Err() case err := <-errc: return err } }
// protoToRecord converts a RequestLog, the internal Protocol Buffer // representation of a single request-level log, to a Record, its // corresponding external representation. func protoToRecord(rl *pb.RequestLog) *Record { offset, err := proto.Marshal(rl.Offset) if err != nil { offset = nil } return &Record{ AppID: *rl.AppId, ModuleID: rl.GetModuleId(), VersionID: *rl.VersionId, RequestID: rl.RequestId, Offset: offset, IP: *rl.Ip, Nickname: rl.GetNickname(), AppEngineRelease: string(rl.GetAppEngineRelease()), StartTime: time.Unix(0, *rl.StartTime*1e3), EndTime: time.Unix(0, *rl.EndTime*1e3), Latency: time.Duration(*rl.Latency) * time.Microsecond, MCycles: *rl.Mcycles, Method: *rl.Method, Resource: *rl.Resource, HTTPVersion: *rl.HttpVersion, Status: *rl.Status, ResponseSize: *rl.ResponseSize, Referrer: rl.GetReferrer(), UserAgent: rl.GetUserAgent(), URLMapEntry: *rl.UrlMapEntry, Combined: *rl.Combined, Host: rl.GetHost(), Cost: rl.GetCost(), TaskQueueName: rl.GetTaskQueueName(), TaskName: rl.GetTaskName(), WasLoadingRequest: rl.GetWasLoadingRequest(), PendingTime: time.Duration(rl.GetPendingTime()) * time.Microsecond, Finished: rl.GetFinished(), AppLogs: protoToAppLogs(rl.Line), InstanceID: string(rl.GetCloneKey()), } }
func (protoCodec) Marshal(v interface{}) ([]byte, error) { return proto.Marshal(v.(proto.Message)) }
func Call(ctx netcontext.Context, service, method string, in, out proto.Message) error { if f, ctx, ok := callOverrideFromContext(ctx); ok { return f(ctx, service, method, in, out) } // Handle already-done contexts quickly. select { case <-ctx.Done(): return ctx.Err() default: } c := fromContext(ctx) if c == nil { // Give a good error message rather than a panic lower down. return errors.New("not an App Engine context") } // Apply transaction modifications if we're in a transaction. if t := transactionFromContext(ctx); t != nil { if t.finished { return errors.New("transaction context has expired") } applyTransaction(in, &t.transaction) } // Default RPC timeout is 60s. timeout := 60 * time.Second if deadline, ok := ctx.Deadline(); ok { timeout = deadline.Sub(time.Now()) } data, err := proto.Marshal(in) if err != nil { return err } ticket := c.req.Header.Get(ticketHeader) req := &remotepb.Request{ ServiceName: &service, Method: &method, Request: data, RequestId: &ticket, } hreqBody, err := proto.Marshal(req) if err != nil { return err } hrespBody, err := c.post(hreqBody, timeout) if err != nil { return err } res := &remotepb.Response{} if err := proto.Unmarshal(hrespBody, res); err != nil { return err } if res.RpcError != nil { ce := &CallError{ Detail: res.RpcError.GetDetail(), Code: *res.RpcError.Code, } switch remotepb.RpcError_ErrorCode(ce.Code) { case remotepb.RpcError_CANCELLED, remotepb.RpcError_DEADLINE_EXCEEDED: ce.Timeout = true } return ce } if res.ApplicationError != nil { return &APIError{ Service: *req.ServiceName, Detail: res.ApplicationError.GetDetail(), Code: *res.ApplicationError.Code, } } if res.Exception != nil || res.JavaException != nil { // This shouldn't happen, but let's be defensive. return &CallError{ Detail: "service bridge returned exception", Code: int32(remotepb.RpcError_UNKNOWN), } } return proto.Unmarshal(res.Response, out) }
func handle(w http.ResponseWriter, req *http.Request) { c := appengine.NewContext(req) u := user.Current(c) if u == nil { u, _ = user.CurrentOAuth(c, "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/appengine.apis", ) } if u == nil || !u.Admin { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusUnauthorized) io.WriteString(w, "You must be logged in as an administrator to access this.\n") return } if req.Header.Get("X-Appcfg-Api-Version") == "" { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusForbidden) io.WriteString(w, "This request did not contain a necessary header.\n") return } if req.Method != "POST" { // Response must be YAML. rtok := req.FormValue("rtok") if rtok == "" { rtok = "0" } w.Header().Set("Content-Type", "text/yaml; charset=utf-8") fmt.Fprintf(w, `{app_id: %q, rtok: %q}`, internal.FullyQualifiedAppID(c), rtok) return } defer req.Body.Close() body, err := ioutil.ReadAll(req.Body) if err != nil { w.WriteHeader(http.StatusBadRequest) log.Errorf(c, "Failed reading body: %v", err) return } remReq := &pb.Request{} if err := proto.Unmarshal(body, remReq); err != nil { w.WriteHeader(http.StatusBadRequest) log.Errorf(c, "Bad body: %v", err) return } service, method := *remReq.ServiceName, *remReq.Method if !requestSupported(service, method) { w.WriteHeader(http.StatusBadRequest) log.Errorf(c, "Unsupported RPC /%s.%s", service, method) return } rawReq := &rawMessage{remReq.Request} rawRes := &rawMessage{} err = internal.Call(c, service, method, rawReq, rawRes) remRes := &pb.Response{} if err == nil { remRes.Response = rawRes.buf } else if ae, ok := err.(*internal.APIError); ok { remRes.ApplicationError = &pb.ApplicationError{ Code: &ae.Code, Detail: &ae.Detail, } } else { // This shouldn't normally happen. log.Errorf(c, "appengine/remote_api: Unexpected error of type %T: %v", err, err) remRes.ApplicationError = &pb.ApplicationError{ Code: proto.Int32(0), Detail: proto.String(err.Error()), } } out, err := proto.Marshal(remRes) if err != nil { // This should not be possible. w.WriteHeader(500) log.Errorf(c, "proto.Marshal: %v", err) return } log.Infof(c, "Spooling %d bytes of response to /%s.%s", len(out), service, method) w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Length", strconv.Itoa(len(out))) w.Write(out) }