func TestReceive(t *testing.T) {
store := store.NewMemStore()
recv := NewReceiver()
recv.MaxOutbox = 100
recv.MaxInbox = 100
recv.FlushInterval = 1
recv.NumOutlets = 2
recv.NumAcceptors = 2
recv.Store = store
recv.Start()
msg := []byte("81 <190>1 2013-03-27T20:02:24+00:00 hostname token shuttle - - measure=hello val=99\n")
recv.Receive("123", msg)
time.Sleep(time.Second * 2)
var buckets []*bucket.Bucket
for bucket := range store.Scan("not important") {
buckets = append(buckets, bucket)
}
if len(buckets) != 1 {
t.FailNow()
}
testBucket := buckets[0]
if testBucket.Id.Name != "hello" {
t.FailNow()
}
if testBucket.Sum() != 99 {
t.FailNow()
}
}
func makeReceiver() (store.Store, *Receiver) {
store := store.NewMemStore()
maxOutbox := 100
maxInbox := 100
recv := NewReceiver(maxInbox, maxOutbox)
recv.NumOutlets = 2
recv.NumAcceptors = 2
recv.Store = store
return store, recv
}
func BenchmarkReceive(b *testing.B) {
b.StopTimer()
st := store.NewMemStore()
recv := NewReceiver(1, 1, time.Second, st)
recv.Start()
defer recv.Stop()
opts := make(map[string][]string)
opts["user"] = []string{"u"}
opts["password"] = []string{"p"}
msg := []byte("94 <190>1 2013-03-27T20:02:24+00:00 hostname token shuttle - - measure.hello=99 measure.world=100")
for i := 0; i < b.N; i++ {
b.StartTimer()
recv.Receive(msg, opts)
b.StopTimer()
b.SetBytes(int64(len(msg)))
}
}
func receiveInput(opts testOps, msg []byte) ([]*bucket.Bucket, error) {
st := store.NewMemStore()
recv := NewReceiver(100, 1, time.Millisecond*5, st)
recv.Start()
defer recv.Stop()
recv.Receive(msg, opts)
time.Sleep(2 * recv.FlushInterval)
future := time.Now().Add(time.Minute)
ch, err := st.Scan(future)
if err != nil {
return nil, err
}
buckets := make([]*bucket.Bucket, 0)
for b := range ch {
buckets = append(buckets, b)
}
return buckets, nil
}
func BenchmarkReceive(b *testing.B) {
b.StopTimer()
store := store.NewMemStore()
maxOutbox := 100
maxInbox := 100
recv := NewReceiver(maxInbox, maxOutbox)
recv.NumOutlets = 10
recv.NumAcceptors = 10
recv.Store = store
recv.FlushInterval = time.Millisecond
recv.Start()
defer recv.Stop()
opts := map[string][]string{}
msg := []byte("81 <190>1 2013-03-27T20:02:24+00:00 hostname token shuttle - - measure=hello val=99\n")
for i := 0; i < b.N; i++ {
b.StartTimer()
recv.Receive("user", "pass", msg, opts)
b.StopTimer()
b.SetBytes(int64(len(msg)))
}
}
func main() {
//The store will be used by receivers and outlets.
var st store.Store
if conf.UsingRedis {
st = store.NewRedisStore(conf.RedisHost,
conf.RedisPass, conf.MaxPartitions, conf.MaxRedisConns)
fmt.Printf("at=initialized-redis-store\n")
} else {
st = store.NewMemStore()
fmt.Printf("at=initialized-mem-store\n")
}
//It is not possible to run both librato and graphite outlets
//in the same process.
switch conf.Outlet {
case "librato":
rdr := outlet.NewBucketReader(conf.BufferSize,
conf.Concurrency, conf.FlushtInterval, st)
outlet := outlet.NewLibratoOutlet(conf.BufferSize,
conf.Concurrency, conf.NumOutletRetry, rdr)
outlet.Start()
if conf.Verbose {
go outlet.Report()
}
case "graphite":
rdr := &outlet.BucketReader{Store: st, Interval: conf.FlushtInterval}
outlet := outlet.NewGraphiteOutlet(conf.BufferSize, rdr)
outlet.Start()
default:
fmt.Println("No outlet running. Run `l2met -h` for outlet help.")
}
//The HTTP Outlet can be ran in addition to the librato or graphite outlet.
if conf.UsingHttpOutlet {
httpOutlet := new(outlet.HttpOutlet)
httpOutlet.Store = st
http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
httpOutlet.ServeReadBucket(w, r)
})
}
if conf.UsingReciever {
recv := receiver.NewReceiver(conf.BufferSize,
conf.Concurrency, conf.FlushtInterval, st)
recv.Start()
if conf.Verbose {
go recv.Report()
}
http.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) {
startReceiveT := time.Now()
if r.Method != "POST" {
http.Error(w, "Invalid Request", 400)
return
}
user, pass, err := auth.Parse(r)
if err != nil {
fmt.Printf("measure.failed-auth erro=%s user=%s pass=%s user-agent=%s token=%s client=%s\n",
err, user, pass, r.Header.Get("User-Agent"), r.Header.Get("Logplex-Drain-Token"), r.Header.Get("X-Forwarded-For"))
http.Error(w, "Invalid Request", 400)
return
}
b, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
http.Error(w, "Invalid Request", 400)
return
}
v := r.URL.Query()
v.Add("user", user)
v.Add("password", pass)
recv.Receive(b, v)
utils.MeasureT("http-receiver", startReceiveT)
})
}
//The only thing that constitutes a healthy l2met
//is the health of the store. In some cases, this might mean
//a redis health check.
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
ok := st.Health()
if !ok {
msg := "Store is unavailable."
fmt.Printf("error=%q\n", msg)
http.Error(w, msg, 500)
}
})
http.HandleFunc("/sign", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Invalid Method. Must be POST.", 400)
return
}
user, pass, err := auth.ParseRaw(r.Header.Get("Authorization"))
if err != nil {
fmt.Printf("measure.failed-auth erro=%s user=%s pass=%s user-agent=%s token=%s client=%s\n",
err, user, pass, r.Header.Get("User-Agent"), r.Header.Get("Logplex-Drain-Token"), r.Header.Get("X-Forwarded-For"))
http.Error(w, "Unable to parse auth headers.", 400)
return
}
matched := false
for i := range conf.Secrets {
if user == conf.Secrets[i] {
matched = true
break
}
}
if !matched {
http.Error(w, "Authentication failed.", 401)
return
}
b, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
http.Error(w, "Unable to read body of POST.", 400)
return
}
signed, err := auth.Sign(b)
if err != nil {
http.Error(w, "Unable to sign body.", 500)
return
}
fmt.Fprint(w, string(signed))
})
//Start the HTTP server.
if e := http.ListenAndServe(fmt.Sprintf(":%d", conf.Port), nil); e != nil {
log.Fatal("Unable to start HTTP server.")
}
fmt.Printf("at=l2met-initialized port=%d\n", conf.Port)
}