Ejemplo n.º 1
0
func init() {
	server, pass, err := utils.ParseRedisUrl()
	if err != nil {
		log.Fatal(err)
	}
	rs = store.NewRedisStore(server, pass, numPartitions, maxRedisConn)
}
Ejemplo n.º 2
0
func main() {
	concurrency := utils.EnvInt("LOCAL_WORKERS", 2)
	numPartitions := utils.EnvUint64("NUM_OUTLET_PARTITIONS", 1)
	server, pass, err := utils.ParseRedisUrl()
	if err != nil {
		log.Fatal(err)
	}
	rs := store.NewRedisStore(server, pass, numPartitions, concurrency*2)

	rdr := outlet.NewBucketReader(1000)
	rdr.NumScanners = concurrency
	rdr.NumOutlets = concurrency
	rdr.Store = rs
	rdr.Interval = time.Millisecond * 500

	l := outlet.NewLibratoOutlet(1000, 1000, 1000)
	l.Reader = rdr
	l.NumOutlets = concurrency
	l.NumConverters = concurrency
	l.Retries = 2
	l.User = utils.EnvString("LIBRATO_USER", "")
	l.Pass = utils.EnvString("LIBRATO_TOKEN", "")
	l.Start()
	select {}
}
Ejemplo n.º 3
0
func main() {
	// The number of partitions that our backends support.
	numPartitions := utils.EnvUint64("NUM_OUTLET_PARTITIONS", 1)
	// The bucket.Store struct will initialize a redis pool for us.
	maxRedisConn := utils.EnvInt("OUTLET_C", 2) + 100
	// We use the store to Put buckets into redis.
	server, pass, err := utils.ParseRedisUrl()
	if err != nil {
		log.Fatal(err)
	}
	rs := store.NewRedisStore(server, pass, numPartitions, maxRedisConn)

	reqBuf := utils.EnvInt("REQUEST_BUFFER", 1000)
	recv := receiver.NewReceiver(reqBuf, reqBuf)
	recv.FlushInterval = time.Millisecond * 200
	recv.NumOutlets = utils.EnvInt("OUTLET_C", 100)
	recv.NumAcceptors = utils.EnvInt("ACCEPT_C", 100)
	recv.Store = rs
	recv.Start()
	go recv.Report()

	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		healthCheck(w, r, rs)
	})
	http.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) {
		recvLogs(w, r, recv)
	})

	httpOutlet := new(outlet.HttpOutlet)
	httpOutlet.Store = rs
	http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
		httpOutlet.ServeReadBucket(w, r)
	})

	port := utils.EnvString("PORT", "8000")
	err = http.ListenAndServe(":"+port, nil)
	if err != nil {
		fmt.Printf("error=%s msg=%q\n", err, "Unable to start http server.")
		os.Exit(1)
	}
	fmt.Printf("at=l2met-initialized port=%s\n", port)
}
Ejemplo n.º 4
0
func main() {
	// The number of partitions that our backends support.
	numPartitions := utils.EnvUint64("NUM_OUTLET_PARTITIONS", 1)
	// The bucket.Store struct will initialize a redis pool for us.
	maxRedisConn := utils.EnvInt("OUTLET_C", 2) + 100
	// We use the store to Put buckets into redis.
	server, pass, err := utils.ParseRedisUrl()
	if err != nil {
		log.Fatal(err)
	}
	rs := store.NewRedisStore(server, pass, numPartitions, maxRedisConn)

	interval := time.Millisecond * 200
	rdr := &outlet.BucketReader{Store: rs, Interval: interval}

	g := outlet.NewGraphiteOutlet()
	g.Reader = rdr
	g.ApiToken = utils.EnvString("GRAPHITE_API_TOKEN", "")
	g.Start()
	select {}
}
Ejemplo n.º 5
0
Archivo: main.go Proyecto: heroku/l2met
func main() {
	//The store will be used by receivers and outlets.
	var st store.Store
	if conf.UsingRedis {
		st = store.NewRedisStore(conf.RedisHost,
			conf.RedisPass, conf.MaxPartitions, conf.MaxRedisConns)
		fmt.Printf("at=initialized-redis-store\n")
	} else {
		st = store.NewMemStore()
		fmt.Printf("at=initialized-mem-store\n")
	}

	//It is not possible to run both librato and graphite outlets
	//in the same process.
	switch conf.Outlet {
	case "librato":
		rdr := outlet.NewBucketReader(conf.BufferSize,
			conf.Concurrency, conf.FlushtInterval, st)
		outlet := outlet.NewLibratoOutlet(conf.BufferSize,
			conf.Concurrency, conf.NumOutletRetry, rdr)
		outlet.Start()
		if conf.Verbose {
			go outlet.Report()
		}
	case "graphite":
		rdr := &outlet.BucketReader{Store: st, Interval: conf.FlushtInterval}
		outlet := outlet.NewGraphiteOutlet(conf.BufferSize, rdr)
		outlet.Start()
	default:
		fmt.Println("No outlet running. Run `l2met -h` for outlet help.")
	}

	//The HTTP Outlet can be ran in addition to the librato or graphite outlet.
	if conf.UsingHttpOutlet {
		httpOutlet := new(outlet.HttpOutlet)
		httpOutlet.Store = st
		http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
			httpOutlet.ServeReadBucket(w, r)
		})
	}

	if conf.UsingReciever {
		recv := receiver.NewReceiver(conf.BufferSize,
			conf.Concurrency, conf.FlushtInterval, st)
		recv.Start()
		if conf.Verbose {
			go recv.Report()
		}
		http.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) {
			startReceiveT := time.Now()
			if r.Method != "POST" {
				http.Error(w, "Invalid Request", 400)
				return
			}
			user, pass, err := auth.Parse(r)
			if err != nil {
				fmt.Printf("measure.failed-auth erro=%s user=%s pass=%s user-agent=%s token=%s client=%s\n",
					err, user, pass, r.Header.Get("User-Agent"), r.Header.Get("Logplex-Drain-Token"), r.Header.Get("X-Forwarded-For"))
				http.Error(w, "Invalid Request", 400)
				return
			}
			b, err := ioutil.ReadAll(r.Body)
			r.Body.Close()
			if err != nil {
				http.Error(w, "Invalid Request", 400)
				return
			}
			v := r.URL.Query()
			v.Add("user", user)
			v.Add("password", pass)
			recv.Receive(b, v)
			utils.MeasureT("http-receiver", startReceiveT)
		})
	}

	//The only thing that constitutes a healthy l2met
	//is the health of the store. In some cases, this might mean
	//a redis health check.
	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		ok := st.Health()
		if !ok {
			msg := "Store is unavailable."
			fmt.Printf("error=%q\n", msg)
			http.Error(w, msg, 500)
		}
	})

	http.HandleFunc("/sign", func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			http.Error(w, "Invalid Method. Must be POST.", 400)
			return
		}
		user, pass, err := auth.ParseRaw(r.Header.Get("Authorization"))
		if err != nil {
			fmt.Printf("measure.failed-auth erro=%s user=%s pass=%s user-agent=%s token=%s client=%s\n",
				err, user, pass, r.Header.Get("User-Agent"), r.Header.Get("Logplex-Drain-Token"), r.Header.Get("X-Forwarded-For"))
			http.Error(w, "Unable to parse auth headers.", 400)
			return
		}
		matched := false
		for i := range conf.Secrets {
			if user == conf.Secrets[i] {
				matched = true
				break
			}
		}
		if !matched {
			http.Error(w, "Authentication failed.", 401)
			return
		}
		b, err := ioutil.ReadAll(r.Body)
		r.Body.Close()
		if err != nil {
			http.Error(w, "Unable to read body of POST.", 400)
			return
		}
		signed, err := auth.Sign(b)
		if err != nil {
			http.Error(w, "Unable to sign body.", 500)
			return
		}
		fmt.Fprint(w, string(signed))
	})

	//Start the HTTP server.
	if e := http.ListenAndServe(fmt.Sprintf(":%d", conf.Port), nil); e != nil {
		log.Fatal("Unable to start HTTP server.")
	}
	fmt.Printf("at=l2met-initialized port=%d\n", conf.Port)
}