Beispiel #1
0
func readConfig(r io.Reader) *rpb.Config {
	raw, err := ioutil.ReadAll(r)
	if err != nil {
		log.Fatal("appengine: could not read from stdin: ", err)
	}

	if len(raw) == 0 {
		// If there were zero bytes, assume this code is not being run as part of
		// a complete app under devappserver2, and generate some reasonable defaults.
		log.Print("appengine: not running under devappserver2; using some default configuration")
		return &rpb.Config{
			AppId:      []byte("dev~my-app"),
			VersionId:  []byte("1.2345"),
			ApiHost:    proto.String("localhost"),
			ApiPort:    proto.Int32(1),
			Datacenter: proto.String("us1"),
			InstanceId: proto.String("deadbeef"),
		}
	}

	b := make([]byte, base64.StdEncoding.DecodedLen(len(raw)))
	n, err := base64.StdEncoding.Decode(b, raw)
	if err != nil {
		log.Fatal("appengine: could not base64 decode stdin: ", err)
	}
	config := &rpb.Config{}

	err = proto.Unmarshal(b[:n], config)
	if err != nil {
		log.Fatal("appengine: could not decode runtime_config: ", err)
	}
	return config
}
Beispiel #2
0
func (cn *Conn) Read(b []byte) (n int, err error) {
	const maxRead = 1 << 20
	if len(b) > maxRead {
		b = b[:maxRead]
	}

	req := &pb.ReceiveRequest{
		SocketDescriptor: &cn.desc,
		DataSize:         proto.Int32(int32(len(b))),
	}
	res := &pb.ReceiveReply{}
	if !cn.readDeadline.IsZero() {
		req.TimeoutSeconds = proto.Float64(cn.readDeadline.Sub(time.Now()).Seconds())
	}
	if err := cn.c.Call("remote_socket", "Receive", req, res, nil); err != nil {
		return 0, err
	}
	if len(res.Data) == 0 {
		return 0, io.EOF
	}
	if len(res.Data) > len(b) {
		return 0, fmt.Errorf("socket: internal error: read too much data: %d > %d", len(res.Data), len(b))
	}
	return copy(b, res.Data), nil
}
Beispiel #3
0
// toRetryParameter converts RetryOptions to pb.TaskQueueRetryParameters.
func (opt *RetryOptions) toRetryParameters() *pb.TaskQueueRetryParameters {
	params := &pb.TaskQueueRetryParameters{}
	if opt.RetryLimit > 0 {
		params.RetryLimit = proto.Int32(opt.RetryLimit)
	}
	if opt.AgeLimit > 0 {
		params.AgeLimitSec = proto.Int64(int64(opt.AgeLimit.Seconds()))
	}
	if opt.MinBackoff > 0 {
		params.MinBackoffSec = proto.Float64(opt.MinBackoff.Seconds())
	}
	if opt.MaxBackoff > 0 {
		params.MaxBackoffSec = proto.Float64(opt.MaxBackoff.Seconds())
	}
	if opt.MaxDoublings > 0 || (opt.MaxDoublings == 0 && opt.ApplyZeroMaxDoublings) {
		params.MaxDoublings = proto.Int32(opt.MaxDoublings)
	}
	return params
}
Beispiel #4
0
// callNext issues a datastore_v3/Next RPC to advance a cursor, such as that
// returned by a query with more results.
func callNext(c appengine.Context, res *pb.QueryResult, offset, limit int32) error {
	if res.Cursor == nil {
		return errors.New("datastore: internal error: server did not return a cursor")
	}
	req := &pb.NextRequest{
		Cursor: res.Cursor,
	}
	if limit >= 0 {
		req.Count = proto.Int32(limit)
	}
	if offset != 0 {
		req.Offset = proto.Int32(offset)
	}
	if res.CompiledCursor != nil {
		req.Compile = proto.Bool(true)
	}
	res.Reset()
	return c.Call("datastore_v3", "Next", req, res, nil)
}
Beispiel #5
0
// Run starts a query for log records, which contain request and application
// level log information.
func (params *Query) Run(c appengine.Context) *Result {
	req := &pb.LogReadRequest{}
	appId := c.FullyQualifiedAppID()
	req.AppId = &appId
	if !params.StartTime.IsZero() {
		req.StartTime = proto.Int64(params.StartTime.UnixNano() / 1e3)
	}
	if !params.EndTime.IsZero() {
		req.EndTime = proto.Int64(params.EndTime.UnixNano() / 1e3)
	}
	if params.Offset != nil {
		var offset pb.LogOffset
		if err := proto.Unmarshal(params.Offset, &offset); err != nil {
			return &Result{context: c, err: fmt.Errorf("bad Offset: %v", err)}
		}
		req.Offset = &offset
	}
	if params.Incomplete {
		req.IncludeIncomplete = &params.Incomplete
	}
	if params.AppLogs {
		req.IncludeAppLogs = &params.AppLogs
	}
	if params.ApplyMinLevel {
		req.MinimumLogLevel = proto.Int32(int32(params.MinLevel))
	}
	if params.Versions == nil {
		// If no versions were specified, default to the major version
		// used by this app.
		versionID := appengine.VersionID(c)
		if i := strings.Index(versionID, "."); i >= 0 {
			versionID = versionID[:i]
		}
		req.VersionId = []string{versionID}
	} else {
		req.VersionId = params.Versions
	}
	if params.RequestIDs != nil {
		ids := make([][]byte, len(params.RequestIDs))
		for i, v := range params.RequestIDs {
			ids[i] = []byte(v)
		}
		req.RequestId = ids
	}

	return &Result{context: c, request: req}
}
Beispiel #6
0
// Get loads the document with the given ID into dst.
//
// The ID is a human-readable ASCII string. It must be non-empty, contain no
// whitespace characters and not start with "!".
//
// dst must be a non-nil struct pointer.
func (x *Index) Get(c appengine.Context, id string, dst interface{}) error {
	if id == "" || !validIndexNameOrDocID(id) {
		return fmt.Errorf("search: invalid ID %q", id)
	}
	req := &pb.ListDocumentsRequest{
		Params: &pb.ListDocumentsParams{
			IndexSpec:  &x.spec,
			StartDocId: proto.String(id),
			Limit:      proto.Int32(1),
		},
	}
	res := &pb.ListDocumentsResponse{}
	if err := c.Call("search", "ListDocuments", req, res, nil); err != nil {
		return err
	}
	if res.Status == nil || res.Status.GetCode() != pb.SearchServiceError_OK {
		return fmt.Errorf("search: %s: %s", res.Status.GetCode(), res.Status.GetErrorDetail())
	}
	if len(res.Document) != 1 || res.Document[0].GetId() != id {
		return ErrNoSuchDocument
	}
	return loadFields(dst, res.Document[0].Field)
}
Beispiel #7
0
// DialTimeout is like Dial but takes a timeout.
// The timeout includes name resolution, if required.
func DialTimeout(c appengine.Context, protocol, addr string, timeout time.Duration) (*Conn, error) {
	var deadline time.Time
	if timeout > 0 {
		deadline = time.Now().Add(timeout)
	}

	host, portStr, err := net.SplitHostPort(addr)
	if err != nil {
		return nil, err
	}
	port, err := strconv.Atoi(portStr)
	if err != nil {
		return nil, fmt.Errorf("socket: bad port %q: %v", portStr, err)
	}

	var prot pb.CreateSocketRequest_SocketProtocol
	switch protocol {
	case "tcp":
		prot = pb.CreateSocketRequest_TCP
	case "udp":
		prot = pb.CreateSocketRequest_UDP
	default:
		return nil, fmt.Errorf("socket: unknown protocol %q", protocol)
	}

	packedAddrs, resolved, err := resolve(c, ipFamilies, host, deadline)
	if err != nil {
		return nil, fmt.Errorf("socket: failed resolving %q: %v", host, err)
	}
	if len(packedAddrs) == 0 {
		return nil, fmt.Errorf("no addresses for %q", host)
	}

	packedAddr := packedAddrs[0] // use first address
	fam := pb.CreateSocketRequest_IPv4
	if len(packedAddr) == net.IPv6len {
		fam = pb.CreateSocketRequest_IPv6
	}

	req := &pb.CreateSocketRequest{
		Family:   fam.Enum(),
		Protocol: prot.Enum(),
		RemoteIp: &pb.AddressPort{
			Port:          proto.Int32(int32(port)),
			PackedAddress: packedAddr,
		},
	}
	if resolved {
		req.RemoteIp.HostnameHint = &host
	}
	res := &pb.CreateSocketReply{}
	if err := c.Call("remote_socket", "CreateSocket", req, res, opts(deadline)); err != nil {
		return nil, err
	}

	return &Conn{
		c:      c,
		desc:   res.GetSocketDescriptor(),
		prot:   prot,
		local:  res.ProxyExternalIp,
		remote: req.RemoteIp,
	}, nil
}
Beispiel #8
0
func handle(w http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)

	if !user.IsAdmin(c) {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusUnauthorized)
		io.WriteString(w, "You must be logged in as an administrator to access this.\n")
		return
	}
	if req.Header.Get("X-Appcfg-Api-Version") == "" {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusForbidden)
		io.WriteString(w, "This request did not contain a necessary header.\n")
		return
	}

	if req.Method != "POST" {
		// Response must be YAML.
		rtok := req.FormValue("rtok")
		if rtok == "" {
			rtok = "0"
		}
		w.Header().Set("Content-Type", "text/yaml; charset=utf-8")
		fmt.Fprintf(w, `{app_id: %q, rtok: %q}`, c.FullyQualifiedAppID(), rtok)
		return
	}

	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		c.Errorf("Failed reading body: %v", err)
		return
	}
	remReq := &pb.Request{}
	if err := proto.Unmarshal(body, remReq); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		c.Errorf("Bad body: %v", err)
		return
	}

	// Only allow datastore_v3 for now, or AllocateIds for datastore_v4.
	service, method := *remReq.ServiceName, *remReq.Method
	ok := service == "datastore_v3" || (service == "datastore_v4" && method == "AllocateIds")
	if !ok {
		w.WriteHeader(http.StatusBadRequest)
		c.Errorf("Unsupported RPC /%s.%s", service, method)
		return
	}

	rawReq := &rawMessage{remReq.Request}
	rawRes := &rawMessage{}
	err = c.Call(service, method, rawReq, rawRes, nil)

	remRes := &pb.Response{}
	if err == nil {
		remRes.Response = rawRes.buf
	} else if ae, ok := err.(*appengine_internal.APIError); ok {
		remRes.ApplicationError = &pb.ApplicationError{
			Code:   &ae.Code,
			Detail: &ae.Detail,
		}
	} else {
		// This shouldn't normally happen.
		c.Errorf("appengine/remote_api: Unexpected error of type %T: %v", err, err)
		remRes.ApplicationError = &pb.ApplicationError{
			Code:   proto.Int32(0),
			Detail: proto.String(err.Error()),
		}
	}
	out, err := proto.Marshal(remRes)
	if err != nil {
		// This should not be possible.
		w.WriteHeader(500)
		c.Errorf("proto.Marshal: %v", err)
		return
	}

	c.Infof("Spooling %d bytes of response to /%s.%s", len(out), service, method)
	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Length", strconv.Itoa(len(out)))
	w.Write(out)
}
Beispiel #9
0
// toProto converts the query to a protocol buffer.
func (q *Query) toProto(dst *pb.Query, appID string) error {
	if q.kind == "" {
		if len(q.filter) != 0 {
			return errors.New("datastore: kindless query cannot have filters")
		}
		if len(q.order) != 0 {
			return errors.New("datastore: kindless query cannot have sort orders")
		}
	}
	if len(q.projection) != 0 && q.keysOnly {
		return errors.New("datastore: query cannot both project and be keys-only")
	}
	dst.Reset()
	dst.App = proto.String(appID)
	if q.kind != "" {
		dst.Kind = proto.String(q.kind)
	}
	if q.ancestor != nil {
		dst.Ancestor = keyToProto(appID, q.ancestor)
	}
	if q.projection != nil {
		dst.PropertyName = q.projection
		if q.distinct {
			dst.GroupByPropertyName = q.projection
		}
	}
	if q.keysOnly {
		dst.KeysOnly = proto.Bool(true)
		dst.RequirePerfectPlan = proto.Bool(true)
	}
	for _, qf := range q.filter {
		if qf.FieldName == "" {
			return errors.New("datastore: empty query filter field name")
		}
		p, errStr := valueToProto(appID, qf.FieldName, reflect.ValueOf(qf.Value), false)
		if errStr != "" {
			return errors.New("datastore: bad query filter value type: " + errStr)
		}
		xf := &pb.Query_Filter{
			Op:       operatorToProto[qf.Op],
			Property: []*pb.Property{p},
		}
		if xf.Op == nil {
			return errors.New("datastore: unknown query filter operator")
		}
		dst.Filter = append(dst.Filter, xf)
	}
	for _, qo := range q.order {
		if qo.FieldName == "" {
			return errors.New("datastore: empty query order field name")
		}
		xo := &pb.Query_Order{
			Property:  proto.String(qo.FieldName),
			Direction: sortDirectionToProto[qo.Direction],
		}
		if xo.Direction == nil {
			return errors.New("datastore: unknown query order direction")
		}
		dst.Order = append(dst.Order, xo)
	}
	if q.limit >= 0 {
		dst.Limit = proto.Int32(q.limit)
	}
	if q.offset != 0 {
		dst.Offset = proto.Int32(q.offset)
	}
	dst.CompiledCursor = q.start
	dst.EndCompiledCursor = q.end
	dst.Compile = proto.Bool(true)
	return nil
}