示例#1
0
文件: id.go 项目: raphweiner/l2met
func (id *Id) Delay(t time.Time) int64 {
	t0 := utils.RoundTime(id.Time, id.Resolution).Unix()
	t1 := utils.RoundTime(t, id.Resolution).Unix()
	base := id.Resolution / time.Second
	if base != 0 {
		return (t1 - t0) / int64(base)
	}
	return 0
}
示例#2
0
文件: main.go 项目: freeformz/l2met
func getBuckets(w http.ResponseWriter, r *http.Request) {
	defer utils.MeasureT(time.Now(), "get-buckets")

	if r.Method != "GET" {
		http.Error(w, "Invalid Request", 400)
		return
	}

	token, err := utils.ParseToken(r)
	if err != nil {
		errmsg := map[string]string{"error": "Missing authorization."}
		utils.WriteJson(w, 401, errmsg)
		return
	}

	q := r.URL.Query()
	limit, err := strconv.ParseInt(q.Get("limit"), 10, 32)
	if err != nil {
		errmsg := map[string]string{"error": "Missing limit parameter."}
		utils.WriteJson(w, 400, errmsg)
		return
	}

	max := utils.RoundTime(time.Now(), time.Minute)
	min := max.Add(-1 * time.Minute * time.Duration(limit))
	buckets, err := store.GetBuckets(token, min, max)
	if err != nil {
		errmsg := map[string]string{"error": "Unable to find buckets"}
		utils.WriteJson(w, 500, errmsg)
		return
	}
	utils.WriteJson(w, 200, buckets)
}
示例#3
0
文件: bucket.go 项目: Epictetus/l2met
func NewBucket(token string, rdr *bufio.Reader) <-chan *Bucket {
	buckets := make(chan *Bucket, 1000)
	go func(c chan<- *Bucket) {
		defer close(c)
		defer utils.MeasureT("new-bucket", time.Now())
		lp := logplex.NewReader(rdr)
		for {
			packet, err := lp.ReadMsg()
			if err != nil {
				if err == io.EOF {
					break
				}
				fmt.Printf("at=logplex-error err=%s\n", err)
				return
			}
			d, err := encoding.ParseMsgData(packet.Msg)
			if err != nil {
				continue
			}

			name, ok := d["measure"]
			if !ok {
				continue
			}

			source, ok := d["source"]
			if !ok {
				source = ""
			}

			var val float64
			tmpVal, ok := d["val"]
			if ok {
				val, err = strconv.ParseFloat(tmpVal, 64)
				if err != nil {
					fmt.Printf("at=error error=\"unable to parse val.\"\n")
					continue
				}
			} else {
				val = float64(1)
			}

			t, err := packet.Time()
			if err != nil {
				fmt.Printf("at=time-error error=%s\n", err)
				continue
			}
			t = utils.RoundTime(t, time.Minute)

			k := BKey{Token: token, Name: name, Source: source, Time: t}
			b := &Bucket{Key: k}
			b.Vals = append(b.Vals, val)
			c <- b
		}
	}(buckets)
	return buckets
}
示例#4
0
文件: bucket.go 项目: freeformz/l2met
func NewBucket(token string, rdr *bufio.Reader) ([]*Bucket, error) {
	var buckets []*Bucket
	lp := logplex.NewReader(rdr)
	for {
		packet, err := lp.ReadMsg()
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Printf("at=logplex-error err=%s\n", err)
			return nil, err
		}

		d, err := encoding.ParseMsgData(packet.Msg)
		if err != nil {
			continue
		}

		name, ok := d["measure"]
		if !ok {
			continue
		}

		source, ok := d["source"]
		if !ok {
			source = ""
		}

		var val float64
		tmpVal, ok := d["val"]
		if ok {
			val, err = strconv.ParseFloat(tmpVal, 64)
			if err != nil {
				fmt.Printf("at=error error=\"unable to parse val.\"\n")
				continue
			}
		} else {
			val = float64(1)
		}

		t, err := packet.Time()
		if err != nil {
			fmt.Printf("at=time-error error=%s\n", err)
			continue
		}

		m := &Bucket{}
		m.Token = token
		m.Time = utils.RoundTime(t, time.Minute)
		m.Name = name
		m.Source = source
		m.Vals = append(m.Vals, val)
		buckets = append(buckets, m)
	}
	utils.MeasureI("received-measurements", int64(len(buckets)))
	return buckets, nil
}
示例#5
0
文件: main.go 项目: Epictetus/l2met
func getMetrics(w http.ResponseWriter, r *http.Request) {
	defer utils.MeasureT("http-metrics", time.Now())

	// Support CORS.
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Headers", "Authorization")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
	if r.Method == "OPTIONS" {
		return
	}

	names := metricsPat.FindStringSubmatch(r.URL.Path)
	if len(names) < 2 {
		fmt.Printf("at=error error=%q\n", "Name parameter not provided.")
		errmsg := map[string]string{"error": "Name parameter not provided."}
		utils.WriteJson(w, 401, errmsg)
		return
	}
	name := names[1]

	token, err := utils.ParseToken(r)
	if err != nil {
		fmt.Printf("at=error error=%q\n", err)
		errmsg := map[string]string{"error": "Missing authorization."}
		utils.WriteJson(w, 401, errmsg)
		return
	}

	q := r.URL.Query()
	limit, err := strconv.ParseInt(q.Get("limit"), 10, 32)
	if err != nil {
		errmsg := map[string]string{"error": "Missing limit parameter."}
		utils.WriteJson(w, 400, errmsg)
		return
	}

	resolution, err := strconv.ParseInt(q.Get("resolution"), 10, 32)
	if err != nil {
		errmsg := map[string]string{"error": "Missing resolution parameter."}
		utils.WriteJson(w, 400, errmsg)
		return
	}

	max := utils.RoundTime(time.Now(), (time.Minute * time.Duration(resolution)))
	min := max.Add(-1 * time.Minute * time.Duration(limit*resolution))

	metrics, err := store.GetMetrics(token, name, resolution, min, max)
	if err != nil {
		errmsg := map[string]string{"error": "Unable to find metrics."}
		utils.WriteJson(w, 500, errmsg)
		return
	}
	utils.WriteJson(w, 200, metrics)
}
示例#6
0
文件: bucket.go 项目: bgentry/l2met
func NewBucket(token string, rdr *bufio.Reader, bSize time.Duration) <-chan *Bucket {
	//TODO(ryandotsmith): Can we eliminate the magical number?
	buckets := make(chan *Bucket, 10000)
	go func(c chan<- *Bucket) {
		defer close(c)
		lp := logplex.NewReader(rdr)
		for {
			packet, err := lp.ReadMsg()
			if err != nil {
				if err == io.EOF {
					break
				}
				fmt.Printf("at=logplex-error err=%s\n", err)
				return
			}
			d, err := encoding.ParseMsgData(packet.Msg)
			if err != nil {
				continue
			}

			measure, ok := d["measure"]
			if !ok {
				continue
			}

			source, ok := d["source"]
			if !ok {
				source = ""
			}

			t, err := packet.Time()
			if err != nil {
				fmt.Printf("at=time-error error=%s\n", err)
				continue
			}
			t = utils.RoundTime(t, bSize)

			val := float64(1)
			tmpVal, present := d["val"]
			if present {
				v, err := strconv.ParseFloat(tmpVal, 64)
				if err == nil {
					val = v
				}
			}

			k := &Id{Token: token, Name: measure, Source: source, Time: t}
			b := &Bucket{Id: k}
			b.Vals = append(b.Vals, val)
			c <- b
		}
	}(buckets)
	return buckets
}
示例#7
0
文件: main.go 项目: freeformz/l2met
// Fetch should kick off the librato outlet process.
// Its responsibility is to get the ids of buckets for the current time,
// make empty Buckets, then place the buckets in an inbox to be filled
// (load the vals into the bucket) and processed.
func fetch(out chan<- *store.Bucket) {
	for _ = range time.Tick(time.Duration(*processInterval) * time.Second) {
		go func(out chan<- *store.Bucket) {
			startPoll := time.Now()
			max := utils.RoundTime(time.Now(), time.Minute)
			min := max.Add(-time.Minute)
			ids, err := allBucketIds(min, max)
			if err != nil {
				utils.MeasureE("find-failed", err)
				return
			}
			for i := range ids {
				b := store.Bucket{Id: ids[i]}
				out <- &b
			}
			utils.MeasureT(startPoll, "librato.fetch")
		}(out)
	}
}
示例#8
0
//TODO(ryandotsmith): NewBucket should be broken up. This func is too big.
func NewBucket(user, pass string, rdr *bufio.Reader, opts map[string][]string) <-chan *Bucket {
	//TODO(ryandotsmith): Can we eliminate the magical number?
	buckets := make(chan *Bucket, 10000)
	go func(c chan<- *Bucket) {
		defer close(c)
		lp := logplex.NewReader(rdr)
		for {
			logLine, err := lp.ReadMsg()
			if err != nil {
				if err == io.EOF {
					break
				}
				fmt.Printf("at=logplex-error err=%s\n", err)
				return
			}

			logData, err := encoding.ParseMsgData(logLine.Msg)
			if err != nil {
				continue
			}

			ts, err := logLine.Time()
			if err != nil {
				fmt.Printf("at=time-error error=%s\n", err)
				continue
			}

			//The resolution determines how long a bucket is
			//left to linger. E.g. a bucket with 1 second resolution
			//will hang around for 1 second and provide metrics
			//with 1 second resolution.
			resQuery, ok := opts["resolution"]
			if !ok {
				resQuery = []string{"60"}
			}
			resTmp, err := strconv.Atoi(resQuery[0])
			if err != nil {
				continue
			}
			res := time.Duration(time.Second * time.Duration(resTmp))
			ts = utils.RoundTime(ts, res)

			//Src can be overridden by the heroku router messages.
			src := logData["source"]

			//You can prefix all measurments by adding the
			//prefix option on your drain url.
			var prefix string
			if prefixQuery, ok := opts["prefix"]; ok {
				if len(prefixQuery[0]) > 0 {
					prefix = prefixQuery[0] + "."
				}
			}

			//Special case the Heroku router.
			//In this case, we will massage logData
			//to include connect, service, and bytes.
			if string(logLine.Pid) == "router" {
				p := "measure.router."
				if len(logData["host"]) > 0 {
					src = logData["host"]
				}
				if len(logData["connect"]) > 0 {
					logData[p+"connect"] = logData["connect"]
				}
				if len(logData["service"]) > 0 {
					logData[p+"service"] = logData["service"]
				}
				if len(logData["bytes"]) > 0 {
					logData[p+"bytes"] = logData["bytes"] + "bytes"
				}
			}

			for k, v := range logData {
				switch k {
				//TODO(ryandotsmith): this case is measre=something val=x
				//It is deprecated and not mentioned in the docs.
				//We should remove this sometime in the near future.
				case "measure":
					units, val := parseVal(logData["val"])
					name := prefix + v
					id := &Id{ts, res, user, pass, name, units, src}
					bucket := &Bucket{Id: id}
					bucket.Vals = []float64{val}
					c <- bucket
				default:
					if !strings.HasPrefix(k, "measure.") {
						break
					}
					name := prefix + k[8:] // len("measure.") == 8
					units, val := parseVal(v)
					id := &Id{ts, res, user, pass, name, units, src}
					bucket := &Bucket{Id: id}
					bucket.Vals = []float64{val}
					c <- bucket
				}
			}
		}
	}(buckets)
	return buckets
}
示例#9
0
func (h *HttpOutlet) ServeReadBucket(w http.ResponseWriter, r *http.Request) {
	// need to extract: token, source, name, time
	// https://l2met:[email protected]/buckets/:name
	user, pass, err := auth.Parse(r)
	if err != nil {
		http.Error(w, "Inavalid Authentication", 401)
		return
	}

	// Shortcut so we can quickly access query params.
	h.Query = r.URL.Query()

	//We need to build the identity of a bucket before we can fetch
	//it from the store. Thus, the following attrs are parsed and held
	//for the bucket.Id.
	src := h.Query.Get("source")
	name := h.Query.Get("name")
	if len(name) == 0 {
		http.Error(w, "Invalid Request. Name is required.", 400)
		return
	}
	res, err := h.parseAssertion("resolution", 60)
	if err != nil {
		http.Error(w, "Invalid Request.", 400)
		return
	}
	resolution := time.Second * time.Duration(res)
	units := h.Query.Get("units")
	if len(units) == 0 {
		units = bucket.DefaultUnit
	}

	//The limit and offset are shortcuts to work with the time
	//field on the bucket. This makes it easy for the client to not have
	//to worry about keeping correct time.
	limit, err := h.parseAssertion("limit", 1)
	if err != nil {
		http.Error(w, "Invalid Request.", 400)
		return
	}
	//The offset is handy because you may not want to take the most recent
	//bucket. For instance, the current minute will not have a complete view
	//of the data; however, the last minute should.
	offset, err := h.parseAssertion("offset", 1)
	if err != nil {
		http.Error(w, "Invalid Request.", 400)
		return
	}

	//The API supports the ability to assert what metrics should be.
	//If the value of the assertion is negative, the assertion can
	//be skipped. By default, the value is negative.
	countAssertion, err := h.parseAssertion("count", -1)
	if err != nil {
		http.Error(w, "Invalid Request.", 400)
		return
	}
	meanAssertion, err := h.parseAssertion("mean", -1)
	if err != nil {
		http.Error(w, "Invalid Request.", 400)
		return
	}
	sumAssertion, err := h.parseAssertion("sum", -1)
	if err != nil {
		http.Error(w, "Invalid Request.", 400)
		return
	}
	//The tolerance is a way to work with assertions that would like to use
	//less than or greater than operators.
	tol, err := h.parseAssertion("tol", 0)
	if err != nil {
		http.Error(w, "Invalid Request.", 400)
		return
	}

	//Build one bucket.Id to share across all the buckets that we fetch
	//with respect to the limit.
	//We will set the time in proceeding for loop.
	id := &bucket.Id{
		User:       user,
		Pass:       pass,
		Name:       name,
		Source:     src,
		Resolution: resolution,
		Units:      units,
	}
	resBucket := &bucket.Bucket{Id: id}
	anchorTime := time.Now()
	for i := 0; i < limit; i++ {
		x := time.Duration((i+offset)*-1) * resolution
		id.Time = utils.RoundTime(anchorTime.Add(x), resolution)
		b := &bucket.Bucket{Id: id}
		//Fetch the bucket from our store.
		//This will fill in the vals.
		h.Store.Get(b)
		//We are only returning 1 bucket from the API. The
		//bucket will contain an aggregate view of the data based on limit.
		resBucket.Add(b)
	}

	//If any of the assertion values are -1 then they were not
	//defined in the request query params. Thus, we only do our assertions
	//if the assertion parameter is > 0.
	if countAssertion > 0 {
		if math.Abs(float64(resBucket.Count()-countAssertion)) > float64(tol) {
			http.Error(w, "Count assertion failed.", 404)
			return
		}
	}
	if meanAssertion > 0 {
		if math.Abs(float64(resBucket.Mean()-float64(meanAssertion))) > float64(tol) {
			http.Error(w, "Mean assertion failed.", 404)
			return
		}
	}
	if sumAssertion > 0 {
		if math.Abs(float64(resBucket.Sum()-float64(sumAssertion))) > float64(tol) {
			http.Error(w, "Sum assertion failed.", 404)
			return
		}
	}

	//Assuming there was not a failed assertion, we can return the result
	//bucket which may contain an aggregate of other buckets via bucket.Add()
	utils.WriteJson(w, 200, resBucket)
}