func (this *alarmWorker) getWindowBorder(wheres ...string) (head, tail int, err error) { query := fmt.Sprintf("SELECT min(ts), max(ts) FROM %s", this.conf.dbName) if len(wheres) > 0 { query += " WHERE 1=1" for _, w := range wheres { query += " AND " + w } } if engine.Globals().Debug { this.project.Println(query) } row := this.db.QueryRow(query) err = row.Scan(&head, &tail) if row == nil || (head == 0 && tail == 0) { err = errSlideWindowEmpty return } if err != nil && engine.Globals().Debug { this.project.Println(head, tail, err) } return }
func (this *DebugOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error { var ( globals = engine.Globals() pack *engine.PipelinePack ok = true inChan = r.InChan() ) LOOP: for ok { select { case pack, ok = <-inChan: if !ok { break LOOP } if !this.blackhole { globals.Println(*pack) } pack.Recycle() } } return nil }
func (this *EsFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error { var ( globals = engine.Globals() pack *engine.PipelinePack ok = true count = 0 inChan = r.InChan() ) LOOP: for ok { select { case pack, ok = <-inChan: if !ok { break LOOP } if globals.Debug { globals.Println(*pack) } if this.handlePack(pack, h.Project(pack.Project)) { count += 1 r.Inject(pack) } else { pack.Recycle() } } } globals.Printf("[%s]Total filtered: %d", r.Name(), count) return nil }
func (this *EsFilter) Init(config *conf.Conf) { this.ident = config.String("ident", "") if this.ident == "" { panic("empty ident") } this.converters = make([]esConverter, 0, 10) this.indexPattern = config.String("index_pattern", "") for i := 0; i < len(config.List("converts", nil)); i++ { section, err := config.Section(fmt.Sprintf("%s[%d]", "converts", i)) if err != nil { panic(err) } c := esConverter{} c.load(section) this.converters = append(this.converters, c) } geodbFile := config.String("geodbfile", "") if err := als.LoadGeoDb(geodbFile); err != nil { panic(err) } globals := engine.Globals() if globals.Verbose { globals.Printf("Loaded geodb %s\n", geodbFile) } }
func (this *CardinalityOutput) handleHttpRequest(w http.ResponseWriter, req *http.Request, params map[string]interface{}) (interface{}, error) { vars := mux.Vars(req) key := vars["key"] globals := engine.Globals() if globals.Verbose { globals.Println(req.Method, key) } output := make(map[string]interface{}) switch req.Method { case "GET": if key == "all" { for _, c := range this.counters.Categories() { output[c] = fmt.Sprintf("[%v] %d", bjtime.TsToString(int(this.counters.StartedAt(c).Unix())), this.counters.Count(c)) } } else { output[key] = this.counters.Count(key) } case "PUT": this.counters.Reset(key) output["msg"] = "ok" } return output, nil }
func (this *NetReceiverInput) handleTcpConnection(conn net.Conn, r engine.InputRunner) { var ( lineReader = bufio.NewReader(conn) line string err error pack *engine.PipelinePack ok bool inChan = r.InChan() globals = engine.Globals() ) globals.Printf("Connection from %s", conn.RemoteAddr()) for { line, err = lineReader.ReadString('\n') if err != nil { globals.Printf("[%s]%s", conn.RemoteAddr(), err) continue } atomic.AddInt64(&this.totalBytes, int64(len(line))) atomic.AddInt64(&this.periodBytes, int64(len(line))) pack, ok = <-inChan if !ok { break } // TODO marshal the pack from line r.Inject(pack) } globals.Printf("Closed connection from %s", conn.RemoteAddr().String()) }
func (this *SkyOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error { var ( ok = true pack *engine.PipelinePack inChan = r.InChan() globals = engine.Globals() project = h.Project(this.project) ) LOOP: for ok { select { case pack, ok = <-inChan: if !ok { break LOOP } if globals.Debug { globals.Println(*pack) } this.feedSky(project, pack) pack.Recycle() } } return nil }
func (this *esBufferWorker) inject(pack *engine.PipelinePack) { this.timestamp = pack.Message.Timestamp switch this.expression { case "count": this.summary.N += 1 default: value, err := pack.Message.FieldValue(this.fieldName, this.fieldType) if err != nil { globals := engine.Globals() if globals.Verbose { globals.Printf("[%s]%v", this.camelName, err) } return } // add counters switch this.fieldType { case als.KEY_TYPE_INT, als.KEY_TYPE_MONEY, als.KEY_TYPE_RANGE: this.summary.Add(float64(value.(int))) case als.KEY_TYPE_FLOAT: this.summary.Add(value.(float64)) } } }
func (this *alarmWorker) insert(args ...interface{}) { if engine.Globals().Debug { this.project.Printf("%s %+v\n", this.conf.title, args) } this.Lock() this.insertStmt.Exec(args...) this.Unlock() }
func (this *NetReceiverInput) reportStats(r engine.InputRunner) { globals := engine.Globals() for _ = range r.Ticker() { globals.Printf("Total %s, speed: %s/s", gofmt.ByteSize(this.totalBytes), gofmt.ByteSize(this.periodBytes)) this.periodBytes = int64(0) } }
func (this *ArchiveInput) Run(r engine.InputRunner, h engine.PluginHelper) error { this.runner = r this.h = h this.chkpnt.Load() go func() { for !this.stopping { select { case <-r.Ticker(): this.chkpnt.Dump() if this.leftN > 0 { engine.Globals().Printf("[%s]Left %d files", r.Name(), this.leftN) } } } }() this.workersWg = new(sync.WaitGroup) filepath.Walk(this.rootDir, this.setLeftN) globals := engine.Globals() if globals.Verbose { globals.Printf("Total files:%d", this.leftN) } // do the real job filepath.Walk(this.rootDir, this.runSingleLogfile) // wait for all workers done this.workersWg.Wait() this.chkpnt.Dump() if globals.Verbose { globals.Printf("[%s]Total msg: %d", r.Name(), this.lineN) } return nil }
// create table schema // for high TPS, each parser has a dedicated sqlite3 db file func (this *alarmWorker) createDB() { const ( DATA_BASEDIR = "data" SQLITE3_DBFILE_SUFFIX = "sqlite" ) dsn := fmt.Sprintf("file:%s?cache=shared&mode=rwc", fmt.Sprintf("%s/%s-%d.%s", DATA_BASEDIR, this.conf.dbName, os.Getpid(), SQLITE3_DBFILE_SUFFIX)) this.db = sqldb.NewSqlDb(sqldb.DRIVER_SQLITE3, dsn, this.project.Logger) this.db.SetDebug(engine.Globals().Debug) this.db.CreateDb(fmt.Sprintf(this.conf.createTable, this.conf.dbName)) }
func (this *SelfSysInput) Run(r engine.InputRunner, h engine.PluginHelper) error { var ( globals = engine.Globals() stats = newSysStat() inChan = r.InChan() pack *engine.PipelinePack jsonString string err error stopped = false ) for !stopped { select { case <-this.stopChan: stopped = true case <-r.Ticker(): // same effect as sleep } if stopped { break } stats.gatherStats() jsonString, err = stats.jsonString() if err != nil { globals.Println(err) continue } pack = <-inChan if err = pack.Message.FromLine(fmt.Sprintf("als,%d,%s", time.Now().Unix(), jsonString)); err != nil { globals.Printf("invalid sys stat: %s\n", jsonString) pack.Recycle() continue } pack.Project = "als" pack.Ident = this.ident pack.EsIndex = "fun_als" pack.EsType = "sys" r.Inject(pack) } return nil }
func (this *alarmWorker) init(config *conf.Conf, stopChan chan interface{}) { this.Mutex = new(sync.Mutex) this.stopChan = stopChan this.history = make(map[string]int64) this.conf = alarmWorkerConfig{} this.conf.init(config) globals := engine.Globals() if this.conf.windowSize.Seconds() < 1.0 { this._instantAlarmOnly = true if this.conf.beepThreshold > 0 { globals.Printf("[%s]instant only alarm needn't set 'beep_threshold'", this.conf.camelName) } if this.conf.abnormalBase != 10 { globals.Printf("[%s]instant only alarm needn't set 'abnormal_base'", this.conf.camelName) } if this.conf.abnormalPercent != 1.5 { globals.Printf("[%s]instant only alarm needn't set 'abnormal_percent'", this.conf.camelName) } if this.conf.showSummary { globals.Printf("[%s]instant only alarm needn't set 'show_summary'", this.conf.camelName) } if this.conf.createTable != "" { globals.Printf("[%s]instant only alarm needn't set 'create_table'", this.conf.camelName) } if this.conf.statsStmt != "" { globals.Printf("[%s]instant only alarm needn't set 'stats_stmt'", this.conf.camelName) } if this.conf.insertStmt != "" { globals.Printf("[%s]instant only alarm needn't set 'insert_stmt'", this.conf.camelName) } if this.conf.printFormat != "" { globals.Printf("[%s]instant only alarm needn't set 'printf'", this.conf.camelName) } if this.conf.dbName != "" { globals.Printf("[%s]instant only alarm needn't set 'dbname'", this.conf.camelName) } } }
func (this *EsBufferFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error { var ( pack *engine.PipelinePack ok = true globals = engine.Globals() inChan = r.InChan() ) for _, worker := range this.wokers { go worker.run(r, h) } LOOP: for ok { select { case pack, ok = <-inChan: if !ok { break LOOP } if globals.Debug { globals.Println(*pack) } this.handlePack(pack) pack.Recycle() } } total := 0 for _, worker := range this.wokers { total += worker.summary.N worker.flush(r, h) } // all workers will get notified and stop running close(this.stopChan) globals.Printf("[%s]Total filtered: %d", r.Name(), total) return nil }
func (this *esBufferWorker) flush(r engine.FilterRunner, h engine.PluginHelper) { if this.summary.N == 0 { return } // generate new pack pack := h.PipelinePack(0) switch this.expression { case "count": pack.Message.SetField(this.esField, this.summary.N) case "mean": pack.Message.SetField(this.esField, this.summary.Mean) case "max": pack.Message.SetField(this.esField, this.summary.Max) case "min": pack.Message.SetField(this.esField, this.summary.Min) case "sd": pack.Message.SetField(this.esField, this.summary.Sd()) case "sum": pack.Message.SetField(this.esField, this.summary.Sum) default: panic("invalid expression: " + this.expression) } pack.Message.Timestamp = this.timestamp pack.Ident = this.ident pack.EsIndex = indexName(h.Project(this.projectName), this.indexPattern, time.Unix(int64(this.timestamp), 0)) pack.EsType = this.esType pack.Project = this.projectName globals := engine.Globals() if globals.Debug { globals.Println(*pack) } r.Inject(pack) this.summary.Reset() }
func (this *NetReceiverInput) Run(r engine.InputRunner, h engine.PluginHelper) error { listener, err := net.Listen("tcp4", this.listenAddr) if err != nil { panic(err) } defer listener.Close() go this.reportStats(r) LOOP: for { conn, err := listener.Accept() if err != nil { engine.Globals().Println(err) break LOOP } go this.handleTcpConnection(conn, r) } return nil }
func (this *EsOutput) showPeriodicalStats() { if !this.showProgress { return } var ( globals = engine.Globals() total = 0 ) globals.Printf("ES types: %d, within %s", this.counters.Len(), this.reportInterval) for _, key := range this.counters.SortedKeys() { val := this.counters.Get(key) if val > 0 { total += val globals.Printf("%-50s %12s", key, gofmt.Comma(int64(val))) this.counters.Set(key, 0) } } globals.Printf("%50s %12s", "Sum", gofmt.Comma(int64(total))) }
// for each inbound pack, this filter will generate several new pack // the original pack will be recycled immediately func (this *CardinalityFilter) handlePack(r engine.FilterRunner, h engine.PluginHelper, pack *engine.PipelinePack) { globals := engine.Globals() for _, c := range this.converters { if !pack.Logfile.MatchPrefix(c.logPrefix) || pack.Project != c.project { continue } for _, f := range c.fields { val, err := pack.Message.FieldValue(f.key, f.typ) if err != nil { if globals.Verbose { h.Project(c.project).Println(err) } return } for _, interval := range f.intervals { // generate new pack p := h.PipelinePack(pack.MsgLoopCount) if p == nil { globals.Println("can't get pack in filter") continue } p.Ident = this.ident p.Project = c.project p.CardinalityKey = fmt.Sprintf("%s.%s.%s", pack.Project, f.key, interval) p.CardinalityData = val p.CardinalityInterval = interval r.Inject(p) } } } }
func (this *EsOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error { var ( pack *engine.PipelinePack reloadChan = make(chan interface{}) ok = true globals = engine.Globals() inChan = r.InChan() reportTicker = time.NewTicker(this.reportInterval) ) this.indexer = core.NewBulkIndexer(this.bulkMaxConn) this.indexer.BulkMaxDocs = this.bulkMaxDocs this.indexer.BulkMaxBuffer = this.bulkMaxBuffer // start the bulk indexer this.indexer.Run(this.stopChan) defer reportTicker.Stop() observer.Subscribe(engine.RELOAD, reloadChan) LOOP: for ok { select { case <-this.stopChan: ok = false case <-reportTicker.C: this.showPeriodicalStats() case <-reloadChan: // TODO case <-time.After(this.flushInterval): this.indexer.Flush() case pack, ok = <-inChan: if !ok { break LOOP } if globals.Debug { globals.Println(*pack) } this.feedEs(h.Project(pack.Project), pack) pack.Recycle() } } engine.Globals().Printf("[%s]Total output to ES: %d", r.Name(), this.totalN) // before shutdown, flush again if globals.Verbose { engine.Globals().Println("Waiting for ES flush...") } this.indexer.Flush() if globals.Verbose { engine.Globals().Println("ES flushed") } // let indexer stop this.stopChan <- true return nil }
func shutdown() { cleanup() globals.Printf("Terminated after %s.", time.Since(engine.Globals().StartedAt)) os.Exit(0) }
func (this *alarmWorker) run(h engine.PluginHelper, goAhead chan bool) { var ( globals = engine.Globals() summary = stats.Summary{} beep bool ever = true ) // lazy assignment this.project = h.Project(this.projName) if globals.DryRun || this._instantAlarmOnly { goAhead <- true return } this.createDB() this.prepareInsertStmt() this.prepareStatsStmt() goAhead <- true for ever { select { case <-time.After(this.conf.windowSize): this.Lock() windowHead, windowTail, err := this.getWindowBorder() if err != nil { this.Unlock() continue } if this.conf.showSummary { summary.Reset() } rows, _ := this.statsStmt.Query(windowTail) cols, _ := rows.Columns() colsN := len(cols) values := make([]interface{}, colsN) valuePtrs := make([]interface{}, colsN) rowSeverity := 0 abnormal := false this.workersMutex.Lock() this.printWindowTitle(windowHead, windowTail, this.conf.title) for rows.Next() { beep = false for i, _ := range cols { valuePtrs[i] = &values[i] } rows.Scan(valuePtrs...) // 1st column always being aggregated quantile var amount = values[0].(int64) if amount == 0 { break } if this.conf.showSummary { summary.Add(float64(amount)) } // beep and feed alarmMail if this.conf.beepThreshold > 0 && int(amount) >= this.conf.beepThreshold { beep = true } rowSeverity = this.conf.severity * int(amount) // abnormal change? blink if this.isAbnormalChange(amount, this.historyKey(this.conf.printFormat, values)) { this.blinkColorPrintfLn(this.conf.printFormat, values...) abnormal = true // multiply factor rowSeverity *= this.conf.abnormalSeverityFactor } this.colorPrintfLn(beep, this.conf.printFormat, values...) this.feedAlarmMail(abnormal, rowSeverity, this.conf.printFormat, values...) } // show summary if this.conf.showSummary && summary.N > 0 { this.colorPrintfLn(false, "Total: %.1f, Mean: %.1f", summary.Sum, summary.Mean) } this.workersMutex.Unlock() rows.Close() this.moveWindowForward(windowTail) this.Unlock() case <-this.stopChan: ever = false } } }
func (this *ArchiveInput) doRunSingleLogfile(path string) { reader := als.NewAlsReader(path) if e := reader.Open(); e != nil { panic(e) } defer func() { reader.Close() this.workersWg.Done() atomic.AddInt32(&this.leftN, -1) <-this.workerNChan // release the lock }() var ( line []byte lineN int inChan = this.runner.InChan() err error project = this.h.Project(this.project) pack *engine.PipelinePack globals = engine.Globals() ) for !this.stopping { line, err = reader.ReadLine() switch err { case nil: lineN += 1 atomic.AddInt64(&this.lineN, 1) if globals.Verbose && lineN == 1 { project.Printf("[%s]started\n", path) } pack = <-inChan if err = pack.Message.FromLine(string(line)); err != nil { if project.ShowError && err != als.ErrEmptyLine { project.Printf("[%s]%v: %s", path, err, string(line)) } pack.Recycle() continue } pack.Ident = this.ident pack.Project = this.project pack.Logfile.SetPath(path) if globals.Debug { globals.Println(*pack) } this.runner.Inject(pack) case io.EOF: if globals.Verbose { project.Printf("[%s]done, lines: %d\n", path, lineN) } this.chkpnt.Put(path) this.chkpnt.Dump() return default: // unknown error panic(err) } } }