// handle a new connection:
// Steps:
// 1. Read input from connection line by line.
// 2. Parse each line into a metric.
// 3. Validate the metric
// 4. Process the metric.
func (d *Detector) handle(conn net.Conn) {
addr := conn.RemoteAddr()
health.IncrNumClients(1)
defer health.DecrNumClients(1)
log.Infof("conn %s established", addr)
scanner := bufio.NewScanner(conn)
for scanner.Scan() { // Read line by line.
if err := scanner.Err(); err != nil { // Close on read error.
log.Errorf("read error: %v, closing conn..", err)
break
}
line := scanner.Text()
m, err := parseMetric(line) // Parse
if err != nil {
log.Errorf("parse error: %v, skipping..", err)
continue
}
if err = m.Validate(); err != nil {
log.Errorf("invalid metric: %v, skipping..", err)
return
}
d.process(m, true)
}
conn.Close()
log.Infof("conn %s disconnected", addr)
}
// Start workers to wait for events.
func (al *Alerter) Start() {
log.Infof("start %d alerter workers..", al.cfg.Alerter.Workers)
for i := 0; i < al.cfg.Alerter.Workers; i++ {
go al.work()
}
al.initAlerterNumsReseter()
}
// work waits for events to alert.
func (al *Alerter) work() {
for {
ev := <-al.In
ew := models.NewWrapperOfEvent(ev) // Avoid locks
if al.checkAlertAt(ew.Metric) { // Check alert interval
continue
}
if al.checkOneDayAlerts(ew.Metric) { // Check one day limit
continue
}
// Avoid noises by issuing alerts only when same alert has occurred
// predefined times.
if al.checkAlertCount(ew.Metric) {
al.setAlertRecord(ew.Metric)
log.Warnf("Not enough alerts with in `AlertCheckInterval` time skipping..: %v", ew.Metric.Name)
continue
}
al.setAlertRecord(ew.Metric)
al.incrAlertNum(ew.Metric)
// Store event
if err := al.storeEvent(ev); err != nil {
log.Warnf("failed to store event:%v, skipping..", err)
continue
}
// Do alert.
var err error
if ew.Project, err = al.getProjByRule(ew.Rule); err != nil {
continue
}
if al.shoudProjBeSilent(ew.Project) {
continue
}
var users []models.User
if users, err = al.getUsersByProj(ew.Project); err != nil {
continue
}
for _, user := range users {
ew.User = &user
if ew.Rule.Level < user.RuleLevel {
continue
}
if len(al.cfg.Alerter.Command) == 0 {
log.Warnf("alert command not configured")
continue
}
if err = al.execCommand(ew); err != nil { // Execute command
log.Errorf("exec %s: %v", al.cfg.Alerter.Command, err)
continue
}
log.Infof("send to %s with %s ok", user.Name, ew.Metric.Name)
}
if len(users) != 0 {
al.setAlertAt(ew.Metric)
health.IncrNumAlertingEvents(1)
}
}
}
// Start the tcp server.
func (d *Detector) Start() {
addr := fmt.Sprintf("0.0.0.0:%d", d.cfg.Detector.Port)
ln, err := net.Listen("tcp", addr)
if err != nil {
log.Fatalf("listen: %v", err)
}
log.Infof("detector is listening on %s", addr)
go d.startIdleTracking()
for {
conn, err := ln.Accept()
if err != nil {
log.Errorf("cannot accept conn: %v, skipping..", err)
continue
}
go d.handle(conn)
}
}
// Start http server.
func Start(c *config.Config, d *storage.DB, f *filter.Filter) {
// Init globals.
cfg = c
db = d
flt = f
// Auth
auth := newAuthHandler(cfg.Webapp.Auth[0], cfg.Webapp.Auth[1])
// Routes
router := httprouter.New()
// Api
router.GET("/api/config", auth.handler(getConfig))
router.GET("/api/interval", getInterval)
router.GET("/api/privateDocUrl", getPrivateDocURL)
router.GET("/api/graphiteUrl", getGraphiteURL)
router.GET("/api/language", getLanguage)
router.GET("/api/projects", getProjects)
router.GET("/api/project/:id", getProject)
router.POST("/api/project", auth.handler(createProject))
router.PATCH("/api/project/:id", auth.handler(updateProject))
router.DELETE("/api/project/:id", auth.handler(deleteProject))
router.GET("/api/project/:id/rules", auth.handler(getProjectRules))
router.GET("/api/project/:id/users", auth.handler(getProjectUsers))
router.POST("/api/project/:id/user", auth.handler(addProjectUser))
router.DELETE("/api/project/:id/user/:user_id", auth.handler(deleteProjectUser))
router.GET("/api/project/:id/events", auth.handler(getEventsByProjectID))
router.GET("/api/users", auth.handler(getUsers))
router.GET("/api/user/:id", auth.handler(getUser))
router.POST("/api/user", auth.handler(createUser))
router.DELETE("/api/user/:id", auth.handler(deleteUser))
router.PATCH("/api/user/:id", auth.handler(updateUser))
router.GET("/api/user/:id/projects", auth.handler(getUserProjects))
router.POST("/api/project/:id/rule", auth.handler(createRule))
router.DELETE("/api/rule/:id", auth.handler(deleteRule))
router.PATCH("/api/rule/:id", auth.handler(editRule))
router.GET("/api/metric/rules/:name", getMetricRules)
router.GET("/api/metric/indexes", getMetricIndexes)
router.GET("/api/metric/data", getMetrics)
router.GET("/api/info", getInfo)
router.GET("/api/version", getVersion)
// Static
router.NotFound = newStaticHandler(http.Dir(cfg.Webapp.Static), auth)
// Serve
addr := fmt.Sprintf("0.0.0.0:%d", cfg.Webapp.Port)
log.Infof("webapp is listening and serving on %s..", addr)
log.Fatal(http.ListenAndServe(addr, router))
}
// createStorage creates a storage for given stamp.
// Dose nothing if the stamp is not large enough.
func (db *DB) createStorage(stamp uint32) error {
id := stamp / db.opts.Period
if len(db.pool) > 0 && id <= db.pool[len(db.pool)-1].id {
// stamp is not large enough.
return nil
}
baseName := strconv.FormatUint(uint64(id), 10)
fileName := path.Join(db.name, baseName)
ldb, err := leveldb.OpenFile(fileName, nil)
if err != nil {
return err
}
s := &storage{db: ldb, id: id}
db.pool = append(db.pool, s)
log.Infof("storage %d created", id)
return nil
}
// createStorage creates a storage for given stamp.
// Dose nothing if the stamp is not large enough.
func (db *DB) createStorage(stamp uint32) error {
id := stamp / db.opts.Period
if len(db.pool) > 0 && id <= db.pool[len(db.pool)-1].id {
// stamp is not large enough.
return nil
}
baseName := strconv.FormatUint(uint64(id), 10)
fileName := path.Join(db.name, baseName)
gdb, err := gorm.Open(dialect, fileName)
if err != nil {
return err
}
s := &storage{db: &gdb, id: id}
if err = s.migrate(); err != nil { // DoNot forget
return err
}
db.pool = append(db.pool, s)
log.Infof("storage %d created", id)
return nil
}
// expireStorage expires storages.
// Dose nothing if the pool needs no expiration.
func (db *DB) expireStorages() error {
if len(db.pool) == 0 {
return nil
}
id := db.pool[len(db.pool)-1].id - db.opts.Expiration/db.opts.Period
pool := db.pool
for i, s := range db.pool {
if s.id < id {
if err := s.close(); err != nil {
return err
}
baseName := strconv.FormatUint(uint64(s.id), 10)
fileName := path.Join(db.name, baseName)
if err := os.RemoveAll(fileName); err != nil {
return err
}
pool = db.pool[i+1:]
log.Infof("storage %d expired", s.id)
}
}
db.pool = pool
return nil
}