Example #1
0
func NewBucket(token string, rdr *bufio.Reader) ([]*Bucket, error) {
	var buckets []*Bucket
	lp := logplex.NewReader(rdr)
	for {
		packet, err := lp.ReadMsg()
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Printf("at=logplex-error err=%s\n", err)
			return nil, err
		}

		d, err := encoding.ParseMsgData(packet.Msg)
		if err != nil {
			continue
		}

		name, ok := d["measure"]
		if !ok {
			continue
		}

		source, ok := d["source"]
		if !ok {
			source = ""
		}

		var val float64
		tmpVal, ok := d["val"]
		if ok {
			val, err = strconv.ParseFloat(tmpVal, 64)
			if err != nil {
				fmt.Printf("at=error error=\"unable to parse val.\"\n")
				continue
			}
		} else {
			val = float64(1)
		}

		t, err := packet.Time()
		if err != nil {
			fmt.Printf("at=time-error error=%s\n", err)
			continue
		}

		m := &Bucket{}
		m.Token = token
		m.Time = utils.RoundTime(t, time.Minute)
		m.Name = name
		m.Source = source
		m.Vals = append(m.Vals, val)
		buckets = append(buckets, m)
	}
	utils.MeasureI("received-measurements", int64(len(buckets)))
	return buckets, nil
}
Example #2
0
func NewBucket(token string, rdr *bufio.Reader) <-chan *Bucket {
	buckets := make(chan *Bucket, 1000)
	go func(c chan<- *Bucket) {
		defer close(c)
		defer utils.MeasureT("new-bucket", time.Now())
		lp := logplex.NewReader(rdr)
		for {
			packet, err := lp.ReadMsg()
			if err != nil {
				if err == io.EOF {
					break
				}
				fmt.Printf("at=logplex-error err=%s\n", err)
				return
			}
			d, err := encoding.ParseMsgData(packet.Msg)
			if err != nil {
				continue
			}

			name, ok := d["measure"]
			if !ok {
				continue
			}

			source, ok := d["source"]
			if !ok {
				source = ""
			}

			var val float64
			tmpVal, ok := d["val"]
			if ok {
				val, err = strconv.ParseFloat(tmpVal, 64)
				if err != nil {
					fmt.Printf("at=error error=\"unable to parse val.\"\n")
					continue
				}
			} else {
				val = float64(1)
			}

			t, err := packet.Time()
			if err != nil {
				fmt.Printf("at=time-error error=%s\n", err)
				continue
			}
			t = utils.RoundTime(t, time.Minute)

			k := BKey{Token: token, Name: name, Source: source, Time: t}
			b := &Bucket{Key: k}
			b.Vals = append(b.Vals, val)
			c <- b
		}
	}(buckets)
	return buckets
}
Example #3
0
func NewBucket(token string, rdr *bufio.Reader, bSize time.Duration) <-chan *Bucket {
	//TODO(ryandotsmith): Can we eliminate the magical number?
	buckets := make(chan *Bucket, 10000)
	go func(c chan<- *Bucket) {
		defer close(c)
		lp := logplex.NewReader(rdr)
		for {
			packet, err := lp.ReadMsg()
			if err != nil {
				if err == io.EOF {
					break
				}
				fmt.Printf("at=logplex-error err=%s\n", err)
				return
			}
			d, err := encoding.ParseMsgData(packet.Msg)
			if err != nil {
				continue
			}

			measure, ok := d["measure"]
			if !ok {
				continue
			}

			source, ok := d["source"]
			if !ok {
				source = ""
			}

			t, err := packet.Time()
			if err != nil {
				fmt.Printf("at=time-error error=%s\n", err)
				continue
			}
			t = utils.RoundTime(t, bSize)

			val := float64(1)
			tmpVal, present := d["val"]
			if present {
				v, err := strconv.ParseFloat(tmpVal, 64)
				if err == nil {
					val = v
				}
			}

			k := &Id{Token: token, Name: measure, Source: source, Time: t}
			b := &Bucket{Id: k}
			b.Vals = append(b.Vals, val)
			c <- b
		}
	}(buckets)
	return buckets
}
Example #4
0
//TODO(ryandotsmith): NewBucket should be broken up. This func is too big.
func NewBucket(user, pass string, rdr *bufio.Reader, opts map[string][]string) <-chan *Bucket {
	//TODO(ryandotsmith): Can we eliminate the magical number?
	buckets := make(chan *Bucket, 10000)
	go func(c chan<- *Bucket) {
		defer close(c)
		lp := logplex.NewReader(rdr)
		for {
			logLine, err := lp.ReadMsg()
			if err != nil {
				if err == io.EOF {
					break
				}
				fmt.Printf("at=logplex-error err=%s\n", err)
				return
			}

			logData, err := encoding.ParseMsgData(logLine.Msg)
			if err != nil {
				continue
			}

			ts, err := logLine.Time()
			if err != nil {
				fmt.Printf("at=time-error error=%s\n", err)
				continue
			}

			//The resolution determines how long a bucket is
			//left to linger. E.g. a bucket with 1 second resolution
			//will hang around for 1 second and provide metrics
			//with 1 second resolution.
			resQuery, ok := opts["resolution"]
			if !ok {
				resQuery = []string{"60"}
			}
			resTmp, err := strconv.Atoi(resQuery[0])
			if err != nil {
				continue
			}
			res := time.Duration(time.Second * time.Duration(resTmp))
			ts = utils.RoundTime(ts, res)

			//Src can be overridden by the heroku router messages.
			src := logData["source"]

			//You can prefix all measurments by adding the
			//prefix option on your drain url.
			var prefix string
			if prefixQuery, ok := opts["prefix"]; ok {
				if len(prefixQuery[0]) > 0 {
					prefix = prefixQuery[0] + "."
				}
			}

			//Special case the Heroku router.
			//In this case, we will massage logData
			//to include connect, service, and bytes.
			if string(logLine.Pid) == "router" {
				p := "measure.router."
				if len(logData["host"]) > 0 {
					src = logData["host"]
				}
				if len(logData["connect"]) > 0 {
					logData[p+"connect"] = logData["connect"]
				}
				if len(logData["service"]) > 0 {
					logData[p+"service"] = logData["service"]
				}
				if len(logData["bytes"]) > 0 {
					logData[p+"bytes"] = logData["bytes"] + "bytes"
				}
			}

			for k, v := range logData {
				switch k {
				//TODO(ryandotsmith): this case is measre=something val=x
				//It is deprecated and not mentioned in the docs.
				//We should remove this sometime in the near future.
				case "measure":
					units, val := parseVal(logData["val"])
					name := prefix + v
					id := &Id{ts, res, user, pass, name, units, src}
					bucket := &Bucket{Id: id}
					bucket.Vals = []float64{val}
					c <- bucket
				default:
					if !strings.HasPrefix(k, "measure.") {
						break
					}
					name := prefix + k[8:] // len("measure.") == 8
					units, val := parseVal(v)
					id := &Id{ts, res, user, pass, name, units, src}
					bucket := &Bucket{Id: id}
					bucket.Vals = []float64{val}
					c <- bucket
				}
			}
		}
	}(buckets)
	return buckets
}