func readConfig(r io.Reader) *rpb.Config {
raw, err := ioutil.ReadAll(r)
if err != nil {
log.Fatal("appengine: could not read from stdin: ", err)
}
if len(raw) == 0 {
// If there were zero bytes, assume this code is not being run as part of
// a complete app under devappserver2, and generate some reasonable defaults.
log.Print("appengine: not running under devappserver2; using some default configuration")
return &rpb.Config{
AppId: []byte("dev~my-app"),
VersionId: []byte("1.2345"),
ApiHost: proto.String("localhost"),
ApiPort: proto.Int32(1),
Datacenter: proto.String("us1"),
InstanceId: proto.String("deadbeef"),
}
}
b := make([]byte, base64.StdEncoding.DecodedLen(len(raw)))
n, err := base64.StdEncoding.Decode(b, raw)
if err != nil {
log.Fatal("appengine: could not base64 decode stdin: ", err)
}
config := &rpb.Config{}
err = proto.Unmarshal(b[:n], config)
if err != nil {
log.Fatal("appengine: could not decode runtime_config: ", err)
}
return config
}
func (cn *Conn) Read(b []byte) (n int, err error) {
const maxRead = 1 << 20
if len(b) > maxRead {
b = b[:maxRead]
}
req := &pb.ReceiveRequest{
SocketDescriptor: &cn.desc,
DataSize: proto.Int32(int32(len(b))),
}
res := &pb.ReceiveReply{}
if !cn.readDeadline.IsZero() {
req.TimeoutSeconds = proto.Float64(cn.readDeadline.Sub(time.Now()).Seconds())
}
if err := cn.c.Call("remote_socket", "Receive", req, res, nil); err != nil {
return 0, err
}
if len(res.Data) == 0 {
return 0, io.EOF
}
if len(res.Data) > len(b) {
return 0, fmt.Errorf("socket: internal error: read too much data: %d > %d", len(res.Data), len(b))
}
return copy(b, res.Data), nil
}
// toRetryParameter converts RetryOptions to pb.TaskQueueRetryParameters.
func (opt *RetryOptions) toRetryParameters() *pb.TaskQueueRetryParameters {
params := &pb.TaskQueueRetryParameters{}
if opt.RetryLimit > 0 {
params.RetryLimit = proto.Int32(opt.RetryLimit)
}
if opt.AgeLimit > 0 {
params.AgeLimitSec = proto.Int64(int64(opt.AgeLimit.Seconds()))
}
if opt.MinBackoff > 0 {
params.MinBackoffSec = proto.Float64(opt.MinBackoff.Seconds())
}
if opt.MaxBackoff > 0 {
params.MaxBackoffSec = proto.Float64(opt.MaxBackoff.Seconds())
}
if opt.MaxDoublings > 0 || (opt.MaxDoublings == 0 && opt.ApplyZeroMaxDoublings) {
params.MaxDoublings = proto.Int32(opt.MaxDoublings)
}
return params
}
// callNext issues a datastore_v3/Next RPC to advance a cursor, such as that
// returned by a query with more results.
func callNext(c appengine.Context, res *pb.QueryResult, offset, limit int32) error {
if res.Cursor == nil {
return errors.New("datastore: internal error: server did not return a cursor")
}
req := &pb.NextRequest{
Cursor: res.Cursor,
}
if limit >= 0 {
req.Count = proto.Int32(limit)
}
if offset != 0 {
req.Offset = proto.Int32(offset)
}
if res.CompiledCursor != nil {
req.Compile = proto.Bool(true)
}
res.Reset()
return c.Call("datastore_v3", "Next", req, res, nil)
}
// Run starts a query for log records, which contain request and application
// level log information.
func (params *Query) Run(c appengine.Context) *Result {
req := &pb.LogReadRequest{}
appId := c.FullyQualifiedAppID()
req.AppId = &appId
if !params.StartTime.IsZero() {
req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3)
}
if !params.EndTime.IsZero() {
req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3)
}
if params.Offset != nil {
var offset pb.LogOffset
if err := proto.Unmarshal(params.Offset, &offset); err != nil {
return &Result{context: c, err: fmt.Errorf("bad Offset: %v", err)}
}
req.Offset = &offset
}
if params.Incomplete {
req.IncludeIncomplete = ¶ms.Incomplete
}
if params.AppLogs {
req.IncludeAppLogs = ¶ms.AppLogs
}
if params.ApplyMinLevel {
req.MinimumLogLevel = proto.Int32(int32(params.MinLevel))
}
if params.Versions == nil {
// If no versions were specified, default to the major version
// used by this app.
versionID := appengine.VersionID(c)
if i := strings.Index(versionID, "."); i >= 0 {
versionID = versionID[:i]
}
req.VersionId = []string{versionID}
} else {
req.VersionId = params.Versions
}
if params.RequestIDs != nil {
ids := make([][]byte, len(params.RequestIDs))
for i, v := range params.RequestIDs {
ids[i] = []byte(v)
}
req.RequestId = ids
}
return &Result{context: c, request: req}
}
// Get loads the document with the given ID into dst.
//
// The ID is a human-readable ASCII string. It must be non-empty, contain no
// whitespace characters and not start with "!".
//
// dst must be a non-nil struct pointer.
func (x *Index) Get(c appengine.Context, id string, dst interface{}) error {
if id == "" || !validIndexNameOrDocID(id) {
return fmt.Errorf("search: invalid ID %q", id)
}
req := &pb.ListDocumentsRequest{
Params: &pb.ListDocumentsParams{
IndexSpec: &x.spec,
StartDocId: proto.String(id),
Limit: proto.Int32(1),
},
}
res := &pb.ListDocumentsResponse{}
if err := c.Call("search", "ListDocuments", req, res, nil); err != nil {
return err
}
if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
}
if len(res.Document) != 1 || res.Document[0].GetId() != id {
return ErrNoSuchDocument
}
return loadFields(dst, res.Document[0].Field)
}
// DialTimeout is like Dial but takes a timeout.
// The timeout includes name resolution, if required.
func DialTimeout(c appengine.Context, protocol, addr string, timeout time.Duration) (*Conn, error) {
var deadline time.Time
if timeout > 0 {
deadline = time.Now().Add(timeout)
}
host, portStr, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portStr)
if err != nil {
return nil, fmt.Errorf("socket: bad port %q: %v", portStr, err)
}
var prot pb.CreateSocketRequest_SocketProtocol
switch protocol {
case "tcp":
prot = pb.CreateSocketRequest_TCP
case "udp":
prot = pb.CreateSocketRequest_UDP
default:
return nil, fmt.Errorf("socket: unknown protocol %q", protocol)
}
packedAddrs, resolved, err := resolve(c, ipFamilies, host, deadline)
if err != nil {
return nil, fmt.Errorf("socket: failed resolving %q: %v", host, err)
}
if len(packedAddrs) == 0 {
return nil, fmt.Errorf("no addresses for %q", host)
}
packedAddr := packedAddrs[0] // use first address
fam := pb.CreateSocketRequest_IPv4
if len(packedAddr) == net.IPv6len {
fam = pb.CreateSocketRequest_IPv6
}
req := &pb.CreateSocketRequest{
Family: fam.Enum(),
Protocol: prot.Enum(),
RemoteIp: &pb.AddressPort{
Port: proto.Int32(int32(port)),
PackedAddress: packedAddr,
},
}
if resolved {
req.RemoteIp.HostnameHint = &host
}
res := &pb.CreateSocketReply{}
if err := c.Call("remote_socket", "CreateSocket", req, res, opts(deadline)); err != nil {
return nil, err
}
return &Conn{
c: c,
desc: res.GetSocketDescriptor(),
prot: prot,
local: res.ProxyExternalIp,
remote: req.RemoteIp,
}, nil
}
func handle(w http.ResponseWriter, req *http.Request) {
c := appengine.NewContext(req)
if !user.IsAdmin(c) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusUnauthorized)
io.WriteString(w, "You must be logged in as an administrator to access this.\n")
return
}
if req.Header.Get("X-Appcfg-Api-Version") == "" {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(http.StatusForbidden)
io.WriteString(w, "This request did not contain a necessary header.\n")
return
}
if req.Method != "POST" {
// Response must be YAML.
rtok := req.FormValue("rtok")
if rtok == "" {
rtok = "0"
}
w.Header().Set("Content-Type", "text/yaml; charset=utf-8")
fmt.Fprintf(w, `{app_id: %q, rtok: %q}`, c.FullyQualifiedAppID(), rtok)
return
}
defer req.Body.Close()
body, err := ioutil.ReadAll(req.Body)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
c.Errorf("Failed reading body: %v", err)
return
}
remReq := &pb.Request{}
if err := proto.Unmarshal(body, remReq); err != nil {
w.WriteHeader(http.StatusBadRequest)
c.Errorf("Bad body: %v", err)
return
}
// Only allow datastore_v3 for now, or AllocateIds for datastore_v4.
service, method := *remReq.ServiceName, *remReq.Method
ok := service == "datastore_v3" || (service == "datastore_v4" && method == "AllocateIds")
if !ok {
w.WriteHeader(http.StatusBadRequest)
c.Errorf("Unsupported RPC /%s.%s", service, method)
return
}
rawReq := &rawMessage{remReq.Request}
rawRes := &rawMessage{}
err = c.Call(service, method, rawReq, rawRes, nil)
remRes := &pb.Response{}
if err == nil {
remRes.Response = rawRes.buf
} else if ae, ok := err.(*appengine_internal.APIError); ok {
remRes.ApplicationError = &pb.ApplicationError{
Code: &ae.Code,
Detail: &ae.Detail,
}
} else {
// This shouldn't normally happen.
c.Errorf("appengine/remote_api: Unexpected error of type %T: %v", err, err)
remRes.ApplicationError = &pb.ApplicationError{
Code: proto.Int32(0),
Detail: proto.String(err.Error()),
}
}
out, err := proto.Marshal(remRes)
if err != nil {
// This should not be possible.
w.WriteHeader(500)
c.Errorf("proto.Marshal: %v", err)
return
}
c.Infof("Spooling %d bytes of response to /%s.%s", len(out), service, method)
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Length", strconv.Itoa(len(out)))
w.Write(out)
}
// toProto converts the query to a protocol buffer.
func (q *Query) toProto(dst *pb.Query, appID string) error {
if q.kind == "" {
if len(q.filter) != 0 {
return errors.New("datastore: kindless query cannot have filters")
}
if len(q.order) != 0 {
return errors.New("datastore: kindless query cannot have sort orders")
}
}
if len(q.projection) != 0 && q.keysOnly {
return errors.New("datastore: query cannot both project and be keys-only")
}
dst.Reset()
dst.App = proto.String(appID)
if q.kind != "" {
dst.Kind = proto.String(q.kind)
}
if q.ancestor != nil {
dst.Ancestor = keyToProto(appID, q.ancestor)
}
if q.projection != nil {
dst.PropertyName = q.projection
if q.distinct {
dst.GroupByPropertyName = q.projection
}
}
if q.keysOnly {
dst.KeysOnly = proto.Bool(true)
dst.RequirePerfectPlan = proto.Bool(true)
}
for _, qf := range q.filter {
if qf.FieldName == "" {
return errors.New("datastore: empty query filter field name")
}
p, errStr := valueToProto(appID, qf.FieldName, reflect.ValueOf(qf.Value), false)
if errStr != "" {
return errors.New("datastore: bad query filter value type: " + errStr)
}
xf := &pb.Query_Filter{
Op: operatorToProto[qf.Op],
Property: []*pb.Property{p},
}
if xf.Op == nil {
return errors.New("datastore: unknown query filter operator")
}
dst.Filter = append(dst.Filter, xf)
}
for _, qo := range q.order {
if qo.FieldName == "" {
return errors.New("datastore: empty query order field name")
}
xo := &pb.Query_Order{
Property: proto.String(qo.FieldName),
Direction: sortDirectionToProto[qo.Direction],
}
if xo.Direction == nil {
return errors.New("datastore: unknown query order direction")
}
dst.Order = append(dst.Order, xo)
}
if q.limit >= 0 {
dst.Limit = proto.Int32(q.limit)
}
if q.offset != 0 {
dst.Offset = proto.Int32(q.offset)
}
dst.CompiledCursor = q.start
dst.EndCompiledCursor = q.end
dst.Compile = proto.Bool(true)
return nil
}
