func (c *remoteContext) call(ctx context.Context, service, method string, in, out proto.Message) error {
req, err := proto.Marshal(in)
if err != nil {
return fmt.Errorf("error marshalling request: %v", err)
}
remReq := &pb.Request{
ServiceName: proto.String(service),
Method: proto.String(method),
Request: req,
// NOTE(djd): RequestId is unused in the server.
}
req, err = proto.Marshal(remReq)
if err != nil {
return fmt.Errorf("proto.Marshal: %v", err)
}
// TODO(djd): Respect ctx.Deadline()?
resp, err := c.client.Post(c.url, "application/octet-stream", bytes.NewReader(req))
if err != nil {
return fmt.Errorf("error sending request: %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad response %d; body: %q", resp.StatusCode, body)
}
if err != nil {
return fmt.Errorf("failed reading response: %v", err)
}
remResp := &pb.Response{}
if err := proto.Unmarshal(body, remResp); err != nil {
return fmt.Errorf("error unmarshalling response: %v", err)
}
if ae := remResp.GetApplicationError(); ae != nil {
return &internal.APIError{
Code: ae.GetCode(),
Detail: ae.GetDetail(),
Service: service,
}
}
if remResp.Response == nil {
return fmt.Errorf("unexpected response: %s", proto.MarshalTextString(remResp))
}
return proto.Unmarshal(remResp.Response, out)
}
// flushLog attempts to flush any pending logs to the appserver.
// It should not be called concurrently.
func (c *context) flushLog(force bool) (flushed bool) {
c.pendingLogs.Lock()
// Grab up to 30 MB. We can get away with up to 32 MB, but let's be cautious.
n, rem := 0, 30<<20
for ; n < len(c.pendingLogs.lines); n++ {
ll := c.pendingLogs.lines[n]
// Each log line will require about 3 bytes of overhead.
nb := proto.Size(ll) + 3
if nb > rem {
break
}
rem -= nb
}
lines := c.pendingLogs.lines[:n]
c.pendingLogs.lines = c.pendingLogs.lines[n:]
c.pendingLogs.Unlock()
if len(lines) == 0 && !force {
// Nothing to flush.
return false
}
rescueLogs := false
defer func() {
if rescueLogs {
c.pendingLogs.Lock()
c.pendingLogs.lines = append(lines, c.pendingLogs.lines...)
c.pendingLogs.Unlock()
}
}()
buf, err := proto.Marshal(&logpb.UserAppLogGroup{
LogLine: lines,
})
if err != nil {
log.Printf("internal.flushLog: marshaling UserAppLogGroup: %v", err)
rescueLogs = true
return false
}
req := &logpb.FlushRequest{
Logs: buf,
}
res := &basepb.VoidProto{}
c.pendingLogs.Lock()
c.pendingLogs.flushes++
c.pendingLogs.Unlock()
if err := Call(toContext(c), "logservice", "Flush", req, res); err != nil {
log.Printf("internal.flushLog: Flush RPC: %v", err)
rescueLogs = true
return false
}
return true
}
// Encode returns an opaque representation of the key
// suitable for use in HTML and URLs.
// This is compatible with the Python and Java runtimes.
func (k *Key) Encode() string {
ref := keyToProto("", k)
b, err := proto.Marshal(ref)
if err != nil {
panic(err)
}
// Trailing padding is stripped.
return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}
// String returns a base-64 string representation of a cursor.
func (c Cursor) String() string {
if c.cc == nil {
return ""
}
b, err := proto.Marshal(c.cc)
if err != nil {
// The only way to construct a Cursor with a non-nil cc field is to
// unmarshal from the byte representation. We panic if the unmarshal
// succeeds but the marshaling of the unchanged protobuf value fails.
panic(fmt.Sprintf("datastore: internal error: malformed cursor: %v", err))
}
return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=")
}
func (c *ProtoClient) Call(ctx context.Context, method string, req, resp proto.Message) error {
payload, err := proto.Marshal(req)
if err != nil {
return err
}
httpReq, err := http.NewRequest("POST", c.endpoint+method, bytes.NewReader(payload))
if err != nil {
return err
}
httpReq.Header.Set("Content-Type", "application/x-protobuf")
if ua := c.userAgent; ua != "" {
httpReq.Header.Set("User-Agent", ua)
}
errc := make(chan error, 1)
cancel := makeReqCancel(httpReq)
go func() {
r, err := c.client.Do(httpReq)
if err != nil {
errc <- err
return
}
defer r.Body.Close()
body, err := ioutil.ReadAll(r.Body)
if r.StatusCode != http.StatusOK {
err = &ErrHTTP{
StatusCode: r.StatusCode,
Body: body,
err: err,
}
}
if err != nil {
errc <- err
return
}
errc <- proto.Unmarshal(body, resp)
}()
select {
case <-ctx.Done():
cancel(c.client.Transport) // Cancel the HTTP request.
return ctx.Err()
case err := <-errc:
return err
}
}
// protoToRecord converts a RequestLog, the internal Protocol Buffer
// representation of a single request-level log, to a Record, its
// corresponding external representation.
func protoToRecord(rl *pb.RequestLog) *Record {
offset, err := proto.Marshal(rl.Offset)
if err != nil {
offset = nil
}
return &Record{
AppID: *rl.AppId,
ModuleID: rl.GetModuleId(),
VersionID: *rl.VersionId,
RequestID: rl.RequestId,
Offset: offset,
IP: *rl.Ip,
Nickname: rl.GetNickname(),
AppEngineRelease: string(rl.GetAppEngineRelease()),
StartTime: time.Unix(0, *rl.StartTime*1e3),
EndTime: time.Unix(0, *rl.EndTime*1e3),
Latency: time.Duration(*rl.Latency) * time.Microsecond,
MCycles: *rl.Mcycles,
Method: *rl.Method,
Resource: *rl.Resource,
HTTPVersion: *rl.HttpVersion,
Status: *rl.Status,
ResponseSize: *rl.ResponseSize,
Referrer: rl.GetReferrer(),
UserAgent: rl.GetUserAgent(),
URLMapEntry: *rl.UrlMapEntry,
Combined: *rl.Combined,
Host: rl.GetHost(),
Cost: rl.GetCost(),
TaskQueueName: rl.GetTaskQueueName(),
TaskName: rl.GetTaskName(),
WasLoadingRequest: rl.GetWasLoadingRequest(),
PendingTime: time.Duration(rl.GetPendingTime()) * time.Microsecond,
Finished: rl.GetFinished(),
AppLogs: protoToAppLogs(rl.Line),
InstanceID: string(rl.GetCloneKey()),
}
}
func (protoCodec) Marshal(v interface{}) ([]byte, error) {
return proto.Marshal(v.(proto.Message))
}
func Call(ctx netcontext.Context, service, method string, in, out proto.Message) error {
if f, ctx, ok := callOverrideFromContext(ctx); ok {
return f(ctx, service, method, in, out)
}
// Handle already-done contexts quickly.
select {
case <-ctx.Done():
return ctx.Err()
default:
}
c := fromContext(ctx)
if c == nil {
// Give a good error message rather than a panic lower down.
return errors.New("not an App Engine context")
}
// Apply transaction modifications if we're in a transaction.
if t := transactionFromContext(ctx); t != nil {
if t.finished {
return errors.New("transaction context has expired")
}
applyTransaction(in, &t.transaction)
}
// Default RPC timeout is 60s.
timeout := 60 * time.Second
if deadline, ok := ctx.Deadline(); ok {
timeout = deadline.Sub(time.Now())
}
data, err := proto.Marshal(in)
if err != nil {
return err
}
ticket := c.req.Header.Get(ticketHeader)
req := &remotepb.Request{
ServiceName: &service,
Method: &method,
Request: data,
RequestId: &ticket,
}
hreqBody, err := proto.Marshal(req)
if err != nil {
return err
}
hrespBody, err := c.post(hreqBody, timeout)
if err != nil {
return err
}
res := &remotepb.Response{}
if err := proto.Unmarshal(hrespBody, res); err != nil {
return err
}
if res.RpcError != nil {
ce := &CallError{
Detail: res.RpcError.GetDetail(),
Code: *res.RpcError.Code,
}
switch remotepb.RpcError_ErrorCode(ce.Code) {
case remotepb.RpcError_CANCELLED, remotepb.RpcError_DEADLINE_EXCEEDED:
ce.Timeout = true
}
return ce
}
if res.ApplicationError != nil {
return &APIError{
Service: *req.ServiceName,
Detail: res.ApplicationError.GetDetail(),
Code: *res.ApplicationError.Code,
}
}
if res.Exception != nil || res.JavaException != nil {
// This shouldn't happen, but let's be defensive.
return &CallError{
Detail: "service bridge returned exception",
Code: int32(remotepb.RpcError_UNKNOWN),
}
}
return proto.Unmarshal(res.Response, out)
}
func handle(w http.ResponseWriter, req *http.Request) {
c := appengine.NewContext(req)
u := user.Current(c)
if u == nil {
u, _ = user.CurrentOAuth(c,
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/appengine.apis",
)
}
if u == nil || !u.Admin {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusUnauthorized)
io.WriteString(w, "You must be logged in as an administrator to access this.\n")
return
}
if req.Header.Get("X-Appcfg-Api-Version") == "" {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusForbidden)
io.WriteString(w, "This request did not contain a necessary header.\n")
return
}
if req.Method != "POST" {
// Response must be YAML.
rtok := req.FormValue("rtok")
if rtok == "" {
rtok = "0"
}
w.Header().Set("Content-Type", "text/yaml; charset=utf-8")
fmt.Fprintf(w, `{app_id: %q, rtok: %q}`, internal.FullyQualifiedAppID(c), rtok)
return
}
defer req.Body.Close()
body, err := ioutil.ReadAll(req.Body)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
log.Errorf(c, "Failed reading body: %v", err)
return
}
remReq := &pb.Request{}
if err := proto.Unmarshal(body, remReq); err != nil {
w.WriteHeader(http.StatusBadRequest)
log.Errorf(c, "Bad body: %v", err)
return
}
service, method := *remReq.ServiceName, *remReq.Method
if !requestSupported(service, method) {
w.WriteHeader(http.StatusBadRequest)
log.Errorf(c, "Unsupported RPC /%s.%s", service, method)
return
}
rawReq := &rawMessage{remReq.Request}
rawRes := &rawMessage{}
err = internal.Call(c, service, method, rawReq, rawRes)
remRes := &pb.Response{}
if err == nil {
remRes.Response = rawRes.buf
} else if ae, ok := err.(*internal.APIError); ok {
remRes.ApplicationError = &pb.ApplicationError{
Code: &ae.Code,
Detail: &ae.Detail,
}
} else {
// This shouldn't normally happen.
log.Errorf(c, "appengine/remote_api: Unexpected error of type %T: %v", err, err)
remRes.ApplicationError = &pb.ApplicationError{
Code: proto.Int32(0),
Detail: proto.String(err.Error()),
}
}
out, err := proto.Marshal(remRes)
if err != nil {
// This should not be possible.
w.WriteHeader(500)
log.Errorf(c, "proto.Marshal: %v", err)
return
}
log.Infof(c, "Spooling %d bytes of response to /%s.%s", len(out), service, method)
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Length", strconv.Itoa(len(out)))
w.Write(out)
}