func GetBuckets(token string, min, max time.Time) ([]*Bucket, error) {
var buckets []*Bucket
startQuery := time.Now()
rows, err := db.PGR.Query("select name, bucket, source, token, vals from metrics where token = $1 and bucket > $2 and bucket <= $3 order by bucket desc",
token, min, max)
if err != nil {
return nil, err
}
utils.MeasureT(startQuery, "buckets.get-all")
startParse := time.Now()
defer rows.Close()
for rows.Next() {
var tmp []byte
b := new(Bucket)
buckets = append(buckets, b)
rows.Scan(&b.Name, &b.Time, &b.Source, &b.Token, &tmp)
if len(tmp) == 0 {
b.Vals = []float64{}
continue
}
encoding.DecodeArray(tmp, &b.Vals)
}
utils.MeasureT(startParse, "buckets.vals.decode")
return buckets, nil
}
func recieveLogs(w http.ResponseWriter, r *http.Request, ch chan<- *store.Bucket) {
defer utils.MeasureT(time.Now(), "http-receiver")
if r.Method != "POST" {
http.Error(w, "Invalid Request", 400)
return
}
defer r.Body.Close()
token, err := utils.ParseToken(r)
if err != nil {
utils.MeasureE("http-auth", err)
http.Error(w, "Invalid Request", 400)
return
}
defer utils.MeasureT(time.Now(), token+"-http-receive")
buckets, err := store.NewBucket(token, bufio.NewReader(r.Body))
if err != nil {
http.Error(w, "Invalid Request", 400)
return
}
for i := range buckets {
ch <- buckets[i]
}
}
func GetMetrics(token, name string, resolution int64, min, max time.Time) ([]*Metric, error) {
startQuery := time.Now()
rows, err := db.PGR.Query("select * from get_metrics($1, $2, $3, $4, $5)",
token, name, resolution, min, max)
if err != nil {
utils.MeasureE("get-metrics-error", err)
return nil, err
}
utils.MeasureT(startQuery, "get-metrics.query")
startParse := time.Now()
defer rows.Close()
var metrics []*Metric
for rows.Next() {
startLoop := time.Now()
var tmp []byte
b := new(Bucket)
rows.Scan(&b.Name, &b.Source, &b.Time, &tmp)
if len(tmp) == 0 {
b.Vals = []float64{}
continue
}
encoding.DecodeArray(tmp, &b.Vals)
m := new(Metric)
m.Time = b.Time
m.Name = b.Name
m.Source = b.Source
m.Mean = b.Mean()
metrics = append(metrics, m)
utils.MeasureT(startLoop, "get-metrics.scan-struct-loop")
}
utils.MeasureT(startParse, "parse.get-metrics")
return metrics, nil
}
func GetMetrics(token, name string, resolution int64, min, max time.Time) ([]*Metric, error) {
defer utils.MeasureT("get-metrics", time.Now())
rows, err := pg.Query("select * from get_buckets($1, $2, $3, $4, $5)",
token, name, resolution, min, max)
if err != nil {
return nil, err
}
defer rows.Close()
var metrics []*Metric
for rows.Next() {
var tmp []byte
k := BKey{}
rows.Scan(&k.Name, &k.Source, &k.Time, &tmp)
b := Bucket{Key: k}
if len(tmp) == 0 {
b.Vals = []float64{}
continue
}
encoding.DecodeArray(tmp, &b.Vals, '{', '}', ',')
m := new(Metric)
m.Time = k.Time
m.Name = k.Name
m.Source = k.Source
m.Mean = b.Mean()
metrics = append(metrics, m)
}
return metrics, nil
}
func ScanBuckets(mailbox string) <-chan *Bucket {
buckets := make(chan *Bucket)
go func(ch chan *Bucket) {
defer utils.MeasureT("redis.scan-buckets", time.Now())
defer close(ch)
rc := redisPool.Get()
defer rc.Close()
rc.Send("MULTI")
rc.Send("SMEMBERS", mailbox)
rc.Send("DEL", mailbox)
reply, err := redis.Values(rc.Do("EXEC"))
if err != nil {
fmt.Printf("at=%q error=%s\n", "redset-smembers", err)
return
}
var delCount int64
var members []string
redis.Scan(reply, &members, &delCount)
for _, member := range members {
k, err := ParseKey(member)
if err != nil {
fmt.Printf("at=parse-key error=%s\n", err)
continue
}
ch <- &Bucket{Key: *k}
}
}(buckets)
return buckets
}
func (b *Bucket) Put(partitions uint64) error {
defer utils.MeasureT("bucket.put", time.Now())
b.Lock()
vals := b.Vals
key := b.String()
partition := b.Partition([]byte(key), partitions)
b.Unlock()
rc := redisPool.Get()
defer rc.Close()
libratoMailBox := fmt.Sprintf("librato_outlet.%d", partition)
//pgMailBox := fmt.Sprintf("postgres_outlet.%d", partition)
rc.Send("MULTI")
rc.Send("RPUSH", key, vals)
rc.Send("EXPIRE", key, 300)
rc.Send("SADD", libratoMailBox, key)
rc.Send("EXPIRE", libratoMailBox, 300)
//rc.Send("SADD", pgMailBox, key)
//rc.Send("EXPIRE", pgMailBox, 300)
_, err := rc.Do("EXEC")
if err != nil {
return err
}
return nil
}
func (s *RedisStore) Putback(partition string, id *bucket.Id) error {
defer utils.MeasureT("bucket.putback", time.Now())
rc := s.redisPool.Get()
defer rc.Close()
_, err := rc.Do("SADD", partition, id.String())
return err
}
func (l *LibratoOutlet) post(u, p string, body *bytes.Buffer) error {
defer utils.MeasureT("librato-post", time.Now())
req, err := http.NewRequest("POST", libratoUrl, body)
if err != nil {
return err
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("User-Agent", "l2met/0")
req.SetBasicAuth(u, p)
resp, err := httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
var m string
s, err := ioutil.ReadAll(resp.Body)
if err != nil {
m = fmt.Sprintf("error=failed-request code=%d", resp.StatusCode)
} else {
m = fmt.Sprintf("error=failed-request code=%d resp=body=%s req-body=%s",
resp.StatusCode, s, body)
}
return errors.New(m)
}
return nil
}
func getBuckets(w http.ResponseWriter, r *http.Request) {
defer utils.MeasureT(time.Now(), "get-buckets")
if r.Method != "GET" {
http.Error(w, "Invalid Request", 400)
return
}
token, err := utils.ParseToken(r)
if err != nil {
errmsg := map[string]string{"error": "Missing authorization."}
utils.WriteJson(w, 401, errmsg)
return
}
q := r.URL.Query()
limit, err := strconv.ParseInt(q.Get("limit"), 10, 32)
if err != nil {
errmsg := map[string]string{"error": "Missing limit parameter."}
utils.WriteJson(w, 400, errmsg)
return
}
max := utils.RoundTime(time.Now(), time.Minute)
min := max.Add(-1 * time.Minute * time.Duration(limit))
buckets, err := store.GetBuckets(token, min, max)
if err != nil {
errmsg := map[string]string{"error": "Unable to find buckets"}
utils.WriteJson(w, 500, errmsg)
return
}
utils.WriteJson(w, 200, buckets)
}
func NewBucket(token string, rdr *bufio.Reader) <-chan *Bucket {
buckets := make(chan *Bucket, 1000)
go func(c chan<- *Bucket) {
defer close(c)
defer utils.MeasureT("new-bucket", time.Now())
lp := logplex.NewReader(rdr)
for {
packet, err := lp.ReadMsg()
if err != nil {
if err == io.EOF {
break
}
fmt.Printf("at=logplex-error err=%s\n", err)
return
}
d, err := encoding.ParseMsgData(packet.Msg)
if err != nil {
continue
}
name, ok := d["measure"]
if !ok {
continue
}
source, ok := d["source"]
if !ok {
source = ""
}
var val float64
tmpVal, ok := d["val"]
if ok {
val, err = strconv.ParseFloat(tmpVal, 64)
if err != nil {
fmt.Printf("at=error error=\"unable to parse val.\"\n")
continue
}
} else {
val = float64(1)
}
t, err := packet.Time()
if err != nil {
fmt.Printf("at=time-error error=%s\n", err)
continue
}
t = utils.RoundTime(t, time.Minute)
k := BKey{Token: token, Name: name, Source: source, Time: t}
b := &Bucket{Key: k}
b.Vals = append(b.Vals, val)
c <- b
}
}(buckets)
return buckets
}
func fetch(t time.Time, outbox chan<- *store.Bucket) {
fmt.Printf("at=start_fetch minute=%d\n", t.Minute())
defer utils.MeasureT("postgres_outlet.fetch", time.Now())
mailbox := fmt.Sprintf("postgres_outlet.%d", partitionId)
for bucket := range store.ScanBuckets(mailbox) {
outbox <- bucket
}
}
func allBucketIds(min, max time.Time) ([]int64, error) {
var buckets []int64
startQuery := time.Now()
r, err := db.PGR.Query("select id from metrics where bucket >= $1 and bucket < $2 order by bucket desc",
min, max)
if err != nil {
return nil, err
}
utils.MeasureT(startQuery, "metrics.query")
startParse := time.Now()
defer r.Close()
for r.Next() {
var id int64
r.Scan(&id)
buckets = append(buckets, id)
}
utils.MeasureT(startParse, "metrics.vals.parse")
return buckets, nil
}
func getMetrics(w http.ResponseWriter, r *http.Request) {
defer utils.MeasureT("http-metrics", time.Now())
// Support CORS.
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Headers", "Authorization")
w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
if r.Method == "OPTIONS" {
return
}
names := metricsPat.FindStringSubmatch(r.URL.Path)
if len(names) < 2 {
fmt.Printf("at=error error=%q\n", "Name parameter not provided.")
errmsg := map[string]string{"error": "Name parameter not provided."}
utils.WriteJson(w, 401, errmsg)
return
}
name := names[1]
token, err := utils.ParseToken(r)
if err != nil {
fmt.Printf("at=error error=%q\n", err)
errmsg := map[string]string{"error": "Missing authorization."}
utils.WriteJson(w, 401, errmsg)
return
}
q := r.URL.Query()
limit, err := strconv.ParseInt(q.Get("limit"), 10, 32)
if err != nil {
errmsg := map[string]string{"error": "Missing limit parameter."}
utils.WriteJson(w, 400, errmsg)
return
}
resolution, err := strconv.ParseInt(q.Get("resolution"), 10, 32)
if err != nil {
errmsg := map[string]string{"error": "Missing resolution parameter."}
utils.WriteJson(w, 400, errmsg)
return
}
max := utils.RoundTime(time.Now(), (time.Minute * time.Duration(resolution)))
min := max.Add(-1 * time.Minute * time.Duration(limit*resolution))
metrics, err := store.GetMetrics(token, name, resolution, min, max)
if err != nil {
errmsg := map[string]string{"error": "Unable to find metrics."}
utils.WriteJson(w, 500, errmsg)
return
}
utils.WriteJson(w, 200, metrics)
}
func CacheGet(c Cachable) ([]byte, bool) {
defer utils.MeasureT(time.Now(), "cache-get")
rc := rp.Get()
defer rc.Close()
bs, err := redis.Bytes(rc.Do("GET", c.Key()))
if err != nil {
utils.MeasureE("redis-get", err)
return nil, false
}
return bs, true
}
func fetch(t time.Time, outbox chan<- *store.Bucket) {
fmt.Printf("at=start_fetch minute=%d\n", t.Minute())
defer utils.MeasureT("postgres_outlet.fetch", time.Now())
pid, err := utils.LockPartition("postgres_outlet", numPartitions, lockTTL)
if err != nil {
log.Fatal("Unable to lock partition.")
}
mailbox := fmt.Sprintf("postgres_outlet.%d", pid)
for bucket := range store.ScanBuckets(mailbox) {
outbox <- bucket
}
}
func (s *RedisStore) putback(id *bucket.Id) error {
defer utils.MeasureT("bucket.putback", time.Now())
rc := s.redisPool.Get()
defer rc.Close()
key := id.String()
partition := s.bucketPartition([]byte(key))
rc.Send("MULTI")
rc.Send("SADD", partition, key)
rc.Send("EXPIRE", partition, 300)
_, err := rc.Do("EXEC")
if err != nil {
return err
}
return nil
}
func DecodeArray(b []byte, dest *[]float64) {
defer utils.MeasureT(time.Now(), "encoding.decode-array")
// pq returns something like: {1.0, 2.0}
// let us remove the { and the }
trimed := b[1:(len(b) - 1)]
// Assuming the numbers are seperated by commas.
numbers := strings.Split(string(trimed), ",")
// Showing that we can do cool things with floats.
for _, x := range numbers {
f, err := strconv.ParseFloat(x, 64)
if err == nil {
*dest = append(*dest, f)
}
}
}
func CacheSet(c Cachable) error {
defer utils.MeasureT(time.Now(), "cache-set")
rc := rp.Get()
defer rc.Close()
bs, err := json.Marshal(c)
if err != nil {
utils.MeasureE("cache-json-encode", err)
return err
}
_, err = rc.Do("SET", c.Key(), bs)
if err != nil {
utils.MeasureE("cache-set", err)
return err
}
return nil
}
// Pull data from the http request, stick it in a channel and close the request.
// We don't do any validation on the data. Always respond with 200.
func recvLogs(w http.ResponseWriter, r *http.Request, recv *receiver.Receiver) {
defer utils.MeasureT("http-receiver", time.Now())
if r.Method != "POST" {
http.Error(w, "Invalid Request", 400)
return
}
token, err := utils.ParseToken(r)
if err != nil {
http.Error(w, "Invalid Request", 400)
return
}
b, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
http.Error(w, "Invalid Request", 400)
return
}
recv.Receive(token, b)
}
// Fetch should kick off the librato outlet process.
// Its responsibility is to get the ids of buckets for the current time,
// make empty Buckets, then place the buckets in an inbox to be filled
// (load the vals into the bucket) and processed.
func fetch(out chan<- *store.Bucket) {
for _ = range time.Tick(time.Duration(*processInterval) * time.Second) {
go func(out chan<- *store.Bucket) {
startPoll := time.Now()
max := utils.RoundTime(time.Now(), time.Minute)
min := max.Add(-time.Minute)
ids, err := allBucketIds(min, max)
if err != nil {
utils.MeasureE("find-failed", err)
return
}
for i := range ids {
b := store.Bucket{Id: ids[i]}
out <- &b
}
utils.MeasureT(startPoll, "librato.fetch")
}(out)
}
}
func (b *Bucket) Get() error {
defer utils.MeasureT("bucket.get", time.Now())
rc := redisPool.Get()
defer rc.Close()
//Fill in the vals.
reply, err := redis.Values(rc.Do("LRANGE", b.String(), 0, -1))
if err != nil {
return err
}
for _, item := range reply {
v, ok := item.([]byte)
if !ok {
continue
}
err = encoding.DecodeArray(v, &b.Vals, '[', ']', ' ')
}
return nil
}
// Pull data from the http request, stick it in a channel and close the request.
// We don't do any validation on the data. Always respond with 200.
func recvLogs(w http.ResponseWriter, r *http.Request, recv *receiver.Receiver) {
defer utils.MeasureT("http-receiver", time.Now())
if r.Method != "POST" {
http.Error(w, "Invalid Request", 400)
return
}
user, pass, err := utils.ParseAuth(r)
if err != nil {
fmt.Printf("measure.failed-auth erro=%s user=%s pass=%s user-agent=%s token=%s client=%s\n",
err, user, pass, r.Header.Get("User-Agent"), r.Header.Get("Logplex-Drain-Token"), r.Header.Get("X-Forwarded-For"))
http.Error(w, "Invalid Request", 400)
return
}
b, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
http.Error(w, "Invalid Request", 400)
return
}
recv.Receive(user, pass, b, r.URL.Query())
}
func receiveLogs(w http.ResponseWriter, r *http.Request, inbox chan<- *LogRequest) {
defer utils.MeasureT("http-receiver", time.Now())
if r.Method != "POST" {
http.Error(w, "Invalid Request", 400)
return
}
token, err := utils.ParseToken(r)
if err != nil {
http.Error(w, "Invalid Request", 400)
return
}
b, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, "Invalid Request", 400)
r.Body.Close()
return
}
r.Body.Close()
inbox <- &LogRequest{token, b}
}
func (b *Bucket) Get() {
defer utils.MeasureT(time.Now(), "bucket.get")
db.PGRLocker.Lock()
rows, err := db.PGR.Query("select name, bucket, source, token, vals from metrics where id = $1",
b.Id)
if err != nil {
fmt.Printf("at=error error=%s\n", err)
return
}
rows.Next()
var tmp []byte
rows.Scan(&b.Name, &b.Time, &b.Source, &b.Token, &tmp)
rows.Close()
db.PGRLocker.Unlock()
if len(tmp) == 0 {
b.Vals = []float64{}
return
}
encoding.DecodeArray(tmp, &b.Vals)
}
func (s *RedisStore) Put(b *bucket.Bucket) error {
defer utils.MeasureT("bucket.put", time.Now())
rc := s.redisPool.Get()
defer rc.Close()
b.Lock()
key := b.Id.String()
value := b.Vals
b.Unlock()
//TODO(ryandotsmith): Ensure consistent keys are being written.
partition := s.bucketPartition("outlet", []byte(key))
rc.Send("MULTI")
rc.Send("RPUSH", key, value)
rc.Send("EXPIRE", key, 300)
rc.Send("SADD", partition, key)
rc.Send("EXPIRE", partition, 300)
_, err := rc.Do("EXEC")
if err != nil {
return err
}
return nil
}
func main() {
//The store will be used by receivers and outlets.
var st store.Store
if conf.UsingRedis {
st = store.NewRedisStore(conf.RedisHost,
conf.RedisPass, conf.MaxPartitions, conf.MaxRedisConns)
fmt.Printf("at=initialized-redis-store\n")
} else {
st = store.NewMemStore()
fmt.Printf("at=initialized-mem-store\n")
}
//It is not possible to run both librato and graphite outlets
//in the same process.
switch conf.Outlet {
case "librato":
rdr := outlet.NewBucketReader(conf.BufferSize,
conf.Concurrency, conf.FlushtInterval, st)
outlet := outlet.NewLibratoOutlet(conf.BufferSize,
conf.Concurrency, conf.NumOutletRetry, rdr)
outlet.Start()
if conf.Verbose {
go outlet.Report()
}
case "graphite":
rdr := &outlet.BucketReader{Store: st, Interval: conf.FlushtInterval}
outlet := outlet.NewGraphiteOutlet(conf.BufferSize, rdr)
outlet.Start()
default:
fmt.Println("No outlet running. Run `l2met -h` for outlet help.")
}
//The HTTP Outlet can be ran in addition to the librato or graphite outlet.
if conf.UsingHttpOutlet {
httpOutlet := new(outlet.HttpOutlet)
httpOutlet.Store = st
http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
httpOutlet.ServeReadBucket(w, r)
})
}
if conf.UsingReciever {
recv := receiver.NewReceiver(conf.BufferSize,
conf.Concurrency, conf.FlushtInterval, st)
recv.Start()
if conf.Verbose {
go recv.Report()
}
http.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) {
startReceiveT := time.Now()
if r.Method != "POST" {
http.Error(w, "Invalid Request", 400)
return
}
user, pass, err := auth.Parse(r)
if err != nil {
fmt.Printf("measure.failed-auth erro=%s user=%s pass=%s user-agent=%s token=%s client=%s\n",
err, user, pass, r.Header.Get("User-Agent"), r.Header.Get("Logplex-Drain-Token"), r.Header.Get("X-Forwarded-For"))
http.Error(w, "Invalid Request", 400)
return
}
b, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
http.Error(w, "Invalid Request", 400)
return
}
v := r.URL.Query()
v.Add("user", user)
v.Add("password", pass)
recv.Receive(b, v)
utils.MeasureT("http-receiver", startReceiveT)
})
}
//The only thing that constitutes a healthy l2met
//is the health of the store. In some cases, this might mean
//a redis health check.
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
ok := st.Health()
if !ok {
msg := "Store is unavailable."
fmt.Printf("error=%q\n", msg)
http.Error(w, msg, 500)
}
})
http.HandleFunc("/sign", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Invalid Method. Must be POST.", 400)
return
}
user, pass, err := auth.ParseRaw(r.Header.Get("Authorization"))
if err != nil {
fmt.Printf("measure.failed-auth erro=%s user=%s pass=%s user-agent=%s token=%s client=%s\n",
err, user, pass, r.Header.Get("User-Agent"), r.Header.Get("Logplex-Drain-Token"), r.Header.Get("X-Forwarded-For"))
http.Error(w, "Unable to parse auth headers.", 400)
return
}
matched := false
for i := range conf.Secrets {
if user == conf.Secrets[i] {
matched = true
break
}
}
if !matched {
http.Error(w, "Authentication failed.", 401)
return
}
b, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
http.Error(w, "Unable to read body of POST.", 400)
return
}
signed, err := auth.Sign(b)
if err != nil {
http.Error(w, "Unable to sign body.", 500)
return
}
fmt.Fprint(w, string(signed))
})
//Start the HTTP server.
if e := http.ListenAndServe(fmt.Sprintf(":%d", conf.Port), nil); e != nil {
log.Fatal("Unable to start HTTP server.")
}
fmt.Printf("at=l2met-initialized port=%d\n", conf.Port)
}