Ejemplo n.º 1
0
// protoToRecord converts a RequestLog, the internal Protocol Buffer
// representation of a single request-level log, to a Record, its
// corresponding external representation.
func protoToRecord(rl *log_proto.RequestLog) *Record {
	return &Record{
		AppID:             *rl.AppId,
		VersionID:         *rl.VersionId,
		RequestID:         *rl.RequestId,
		IP:                *rl.Ip,
		Nickname:          proto.GetString(rl.Nickname),
		StartTime:         *rl.StartTime,
		EndTime:           *rl.EndTime,
		Latency:           *rl.Latency,
		MCycles:           *rl.Mcycles,
		Method:            *rl.Method,
		Resource:          *rl.Resource,
		HTTPVersion:       *rl.HttpVersion,
		Status:            *rl.Status,
		ResponseSize:      *rl.ResponseSize,
		Referrer:          proto.GetString(rl.Referrer),
		UserAgent:         proto.GetString(rl.UserAgent),
		URLMapEntry:       *rl.UrlMapEntry,
		Combined:          *rl.Combined,
		APIMCycles:        proto.GetInt64(rl.ApiMcycles),
		Host:              proto.GetString(rl.Host),
		Cost:              proto.GetFloat64(rl.Cost),
		TaskQueueName:     proto.GetString(rl.TaskQueueName),
		TaskName:          proto.GetString(rl.TaskName),
		WasLoadingRequest: proto.GetBool(rl.WasLoadingRequest),
		PendingTime:       proto.GetInt64(rl.PendingTime),
		Finished:          proto.GetBool(rl.Finished),
		AppLogs:           protoToAppLogs(rl.Line),
	}
}
Ejemplo n.º 2
0
func (t *Iterator) next() (*Key, *pb.EntityProto, error) {
	if t.err != nil {
		return nil, nil, t.err
	}

	// Issue datastore_v3/Next RPCs as necessary.
	for len(t.res.Result) == 0 {
		if !proto.GetBool(t.res.MoreResults) {
			t.err = Done
			return nil, nil, t.err
		}
		t.offset -= proto.GetInt32(t.res.SkippedResults)
		if t.offset < 0 {
			t.offset = 0
		}
		if err := callNext(t.c, &t.res, t.offset, t.limit, zeroLimitMeansUnlimited); err != nil {
			t.err = err
			return nil, nil, t.err
		}
		// For an Iterator, a zero limit means unlimited.
		if t.limit == 0 {
			continue
		}
		t.limit -= int32(len(t.res.Result))
		if t.limit > 0 {
			continue
		}
		t.limit = 0
		if proto.GetBool(t.res.MoreResults) {
			t.err = errors.New("datastore: internal error: limit exhausted but more_results is true")
			return nil, nil, t.err
		}
	}

	// Pop the EntityProto from the front of t.res.Result and
	// extract its key.
	var e *pb.EntityProto
	e, t.res.Result = t.res.Result[0], t.res.Result[1:]
	if e.Key == nil {
		return nil, nil, errors.New("datastore: internal error: server did not return a key")
	}
	k, err := protoToKey(e.Key)
	if err != nil || k.Incomplete() {
		return nil, nil, errors.New("datastore: internal error: server returned an invalid key")
	}
	if proto.GetBool(t.res.KeysOnly) {
		return k, nil, nil
	}
	return k, e, nil
}
Ejemplo n.º 3
0
func (t *Iterator) next() (*Key, *pb.EntityProto, os.Error) {
	if t.err != nil {
		return nil, nil, t.err
	}

	// Issue an RPC if necessary.
	call := false
	if t.res == nil {
		call = true
		t.res = &pb.QueryResult{}
	} else if len(t.res.Result) == 0 {
		if !proto.GetBool(t.res.MoreResults) {
			t.err = Done
			return nil, nil, t.err
		}
		call = true
		t.res.Reset()
	}
	if call {
		if t.offset != 0 {
			if t.offset < 0 || t.offset > math.MaxInt32 {
				t.err = os.NewError("datastore: query offset overflow")
				return nil, nil, t.err
			}
			if t.req.Offset == nil {
				t.req.Offset = new(int32)
			}
			*t.req.Offset = int32(t.offset)
		}
		t.err = t.c.Call("datastore_v3", "RunQuery", t.req, t.res)
		if t.err != nil {
			return nil, nil, t.err
		}
	}
	if len(t.res.Result) == 0 {
		// TODO: This code is probably broken for
		// queries with offset > 1000.
		t.err = Done
		return nil, nil, t.err
	}
	t.offset++

	// Pop the EntityProto from the front of t.res.Result and
	// extract its key.
	var e *pb.EntityProto
	e, t.res.Result = t.res.Result[0], t.res.Result[1:]
	if e.Key == nil {
		return nil, nil, os.NewError("datastore: internal error: server did not return a key")
	}
	k, err := protoToKey(e.Key)
	if err != nil || k.Incomplete() {
		return nil, nil, os.NewError("datastore: internal error: server returned an invalid key")
	}
	if t.keysOnly {
		return k, nil, nil
	}
	return k, e, nil
}
Ejemplo n.º 4
0
// loadMapEntry converts a Property into an entry of an existing Map,
// or into an element of a slice-valued Map entry.
func loadMapEntry(m Map, k *Key, p *pb.Property) os.Error {
	var (
		result    interface{}
		sliceType reflect.Type
	)
	switch {
	case p.Value.Int64Value != nil:
		if p.Meaning != nil && *p.Meaning == pb.Property_GD_WHEN {
			result = Time(*p.Value.Int64Value)
			sliceType = reflect.TypeOf([]Time(nil))
		} else {
			result = *p.Value.Int64Value
			sliceType = reflect.TypeOf([]int64(nil))
		}
	case p.Value.BooleanValue != nil:
		result = *p.Value.BooleanValue
		sliceType = reflect.TypeOf([]bool(nil))
	case p.Value.StringValue != nil:
		if p.Meaning != nil && *p.Meaning == pb.Property_BLOB {
			result = []byte(*p.Value.StringValue)
			sliceType = reflect.TypeOf([][]byte(nil))
		} else if p.Meaning != nil && *p.Meaning == pb.Property_BLOBKEY {
			result = appengine.BlobKey(*p.Value.StringValue)
			sliceType = reflect.TypeOf([]appengine.BlobKey(nil))
		} else {
			result = *p.Value.StringValue
			sliceType = reflect.TypeOf([]string(nil))
		}
	case p.Value.DoubleValue != nil:
		result = *p.Value.DoubleValue
		sliceType = reflect.TypeOf([]float64(nil))
	case p.Value.Referencevalue != nil:
		key, err := referenceValueToKey(p.Value.Referencevalue)
		if err != nil {
			return err
		}
		result = key
		sliceType = reflect.TypeOf([]*Key(nil))
	default:
		return nil
	}
	name := proto.GetString(p.Name)
	if proto.GetBool(p.Multiple) {
		var s reflect.Value
		if x := m[name]; x != nil {
			s = reflect.ValueOf(x)
		} else {
			s = reflect.MakeSlice(sliceType, 0, 0)
		}
		s = reflect.Append(s, reflect.ValueOf(result))
		m[name] = s.Interface()
	} else {
		m[name] = result
	}
	return nil
}
Ejemplo n.º 5
0
Archivo: nagios.go Proyecto: pjjw/ncd
func (c *CheckResult) stringMap() (smap map[string]string) {
	smap = map[string]string{
		"Hostname":    fmt.Sprintf("%s", proto.GetString(c.Hostname)),
		"ServiceName": fmt.Sprintf("%s", proto.GetString(c.ServiceName)),
		"Status":      fmt.Sprintf("%d", int32((*c.Status))),
		"CheckPassive": fmt.Sprintf("%d", func() (i int32) {
			if proto.GetBool(c.CheckPassive) {
				i = 1
			} else {
				i = 0
			}
			return i
		}()),
		"CheckOutput":    fmt.Sprintf("%s", strings.Trim(strconv.Quote(proto.GetString(c.CheckOutput)), "\"")),
		"StartTimestamp": fmt.Sprintf("%f", float64(proto.GetInt64(c.StartTimestamp))/1000000000),
		"EndTimestamp":   fmt.Sprintf("%f", float64(proto.GetInt64(c.EndTimestamp))/1000000000),
		"TimeNow":        fmt.Sprintf("%d", time.Seconds()),
	}
	return smap
}
Ejemplo n.º 6
0
// RoundTrip issues a single HTTP request and returns its response. Per the
// http.RoundTripper interface, RoundTrip only returns an error if there
// was a problem with the request being malformed
// (ErrInvalidFetchRequest) or the URL Fetch proxy fails (ErrFetch).
// Note that HTTP response codes such as 5xx, 403, 404, etc are not
// errors as far as the transport is concerned and will be returned
// with err set to nil.
func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err os.Error) {
	methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method]
	if !ok {
		return nil, &ErrInvalidFetchRequest{"Unsupported method: " + req.Method, nil}
	}

	method := pb.URLFetchRequest_RequestMethod(methNum)

	freq := &pb.URLFetchRequest{
		Method:                        &method,
		Url:                           proto.String(req.URL.String()),
		FollowRedirects:               proto.Bool(false), // http.Client's responsibility
		MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate),
	}

	if t.DeadlineSeconds != 0 {
		freq.Deadline = proto.Float64(t.DeadlineSeconds)
	}

	for k, vals := range req.Header {
		for _, val := range vals {
			freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{
				Key:   proto.String(k),
				Value: proto.String(val),
			})
		}
	}
	if methodAcceptsRequestBody[req.Method] {
		freq.Payload, err = ioutil.ReadAll(req.Body)
		if err != nil {
			return nil, &ErrInvalidFetchRequest{"Failed to read body", err}
		}
	}

	fres := &pb.URLFetchResponse{}
	if err := t.Context.Call("urlfetch", "Fetch", freq, fres); err != nil {
		return nil, &ErrFetch{err.String()}
	}

	res = &http.Response{}
	res.StatusCode = int(*fres.StatusCode)
	res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode))
	res.Header = http.Header(make(map[string][]string))
	res.RequestMethod = req.Method

	// Faked:
	res.ProtoMajor = 1
	res.ProtoMinor = 1
	res.Proto = "HTTP/1.1"
	res.Close = true

	for _, h := range fres.Header {
		hkey := http.CanonicalHeaderKey(*h.Key)
		hval := *h.Value
		if hkey == "Content-Length" {
			// Will get filled in below for all but HEAD requests.
			if req.Method == "HEAD" {
				res.ContentLength, _ = strconv.Atoi64(hval)
			}
			continue
		}
		res.Header.Add(hkey, hval)
	}

	if req.Method != "HEAD" {
		res.ContentLength = int64(len(fres.Content))
	}

	truncated := proto.GetBool(fres.ContentWasTruncated)
	res.Body = &bodyReader{content: fres.Content, truncated: truncated}
	return
}
Ejemplo n.º 7
0
// loadStructField converts a Property into a field of an existing struct,
// or into an element of a slice-typed struct field.
// It returns an error message, or "" for success.
func loadStructField(sv reflect.Value, p *pb.Property) string {
	fieldName := proto.GetString(p.Name)
	v := sv.FieldByName(fieldName)
	if !v.IsValid() {
		return "no such struct field"
	}
	if unexported(fieldName) {
		return "unexported struct field"
	}
	var slice reflect.Value
	if proto.GetBool(p.Multiple) {
		if v.Kind() != reflect.Slice {
			return "multiple-valued property requires a slice field type"
		}
		if v.Len() > maxSliceFieldLen-1 {
			return "slice is too long"
		}
		slice = v
		v = reflect.New(v.Type().Elem()).Elem()
	}
	switch v.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if p.Value.Int64Value == nil {
			return typeMismatchReason(p, v)
		}
		x := *p.Value.Int64Value
		if v.OverflowInt(x) {
			return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
		}
		v.SetInt(x)
	case reflect.Bool:
		if p.Value.BooleanValue == nil {
			return typeMismatchReason(p, v)
		}
		v.SetBool(*p.Value.BooleanValue)
	case reflect.String:
		if p.Value.StringValue == nil {
			return typeMismatchReason(p, v)
		}
		v.SetString(*p.Value.StringValue)
	case reflect.Float32, reflect.Float64:
		if p.Value.DoubleValue == nil {
			return typeMismatchReason(p, v)
		}
		x := *p.Value.DoubleValue
		if v.OverflowFloat(x) {
			return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
		}
		v.SetFloat(x)
	case reflect.Ptr:
		if _, ok := v.Interface().(*Key); !ok {
			return typeMismatchReason(p, v)
		}
		if p.Value.Referencevalue == nil {
			return typeMismatchReason(p, v)
		}
		k, err := referenceValueToKey(p.Value.Referencevalue)
		if err != nil {
			return "stored key was invalid"
		}
		v.Set(reflect.ValueOf(k))
	case reflect.Slice:
		if _, ok := v.Interface().([]byte); !ok {
			return typeMismatchReason(p, v)
		}
		if p.Value.StringValue == nil {
			return typeMismatchReason(p, v)
		}
		b := []byte(*p.Value.StringValue)
		v.Set(reflect.ValueOf(b))
	default:
		return typeMismatchReason(p, v)
	}
	if slice.IsValid() {
		slice.Set(reflect.Append(slice, v))
	}
	return ""
}
Ejemplo n.º 8
0
// Count returns the number of results for the query.
func (q *Query) Count(c appengine.Context) (int, error) {
	// Check that the query is well-formed.
	if q.err != nil {
		return 0, q.err
	}

	// Run a copy of the query, with keysOnly true, and an adjusted offset.
	// We also set the limit to zero, as we don't want any actual entity data,
	// just the number of skipped results.
	newQ := *q
	newQ.keysOnly = true
	newQ.limit = 0
	if q.limit == 0 {
		// If the original query was unlimited, set the new query's offset to maximum.
		newQ.offset = math.MaxInt32
	} else {
		newQ.offset = q.offset + q.limit
		if newQ.offset < 0 {
			// Do the best we can, in the presence of overflow.
			newQ.offset = math.MaxInt32
		}
	}
	req := &pb.Query{}
	if err := newQ.toProto(req, c.FullyQualifiedAppID(), zeroLimitMeansZero); err != nil {
		return 0, err
	}
	res := &pb.QueryResult{}
	if err := c.Call("datastore_v3", "RunQuery", req, res, nil); err != nil {
		return 0, err
	}

	// n is the count we will return. For example, suppose that our original
	// query had an offset of 4 and a limit of 2008: the count will be 2008,
	// provided that there are at least 2012 matching entities. However, the
	// RPCs will only skip 1000 results at a time. The RPC sequence is:
	//   call RunQuery with (offset, limit) = (2012, 0)  // 2012 == newQ.offset
	//   response has (skippedResults, moreResults) = (1000, true)
	//   n += 1000  // n == 1000
	//   call Next     with (offset, limit) = (1012, 0)  // 1012 == newQ.offset - n
	//   response has (skippedResults, moreResults) = (1000, true)
	//   n += 1000  // n == 2000
	//   call Next     with (offset, limit) = (12, 0)    // 12 == newQ.offset - n
	//   response has (skippedResults, moreResults) = (12, false)
	//   n += 12    // n == 2012
	//   // exit the loop
	//   n -= 4     // n == 2008
	var n int32
	for {
		// The QueryResult should have no actual entity data, just skipped results.
		if len(res.Result) != 0 {
			return 0, errors.New("datastore: internal error: Count request returned too much data")
		}
		n += proto.GetInt32(res.SkippedResults)
		if !proto.GetBool(res.MoreResults) {
			break
		}
		if err := callNext(c, res, newQ.offset-n, 0, zeroLimitMeansZero); err != nil {
			return 0, err
		}
	}
	n -= q.offset
	if n < 0 {
		// If the offset was greater than the number of matching entities,
		// return 0 instead of negative.
		n = 0
	}
	return int(n), nil
}