func (this *alarmWorker) getWindowBorder(wheres ...string) (head, tail int, err error) {
query := fmt.Sprintf("SELECT min(ts), max(ts) FROM %s", this.conf.dbName)
if len(wheres) > 0 {
query += " WHERE 1=1"
for _, w := range wheres {
query += " AND " + w
}
}
if engine.Globals().Debug {
this.project.Println(query)
}
row := this.db.QueryRow(query)
err = row.Scan(&head, &tail)
if row == nil || (head == 0 && tail == 0) {
err = errSlideWindowEmpty
return
}
if err != nil && engine.Globals().Debug {
this.project.Println(head, tail, err)
}
return
}
func (this *DebugOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
var (
globals = engine.Globals()
pack *engine.PipelinePack
ok = true
inChan = r.InChan()
)
LOOP:
for ok {
select {
case pack, ok = <-inChan:
if !ok {
break LOOP
}
if !this.blackhole {
globals.Println(*pack)
}
pack.Recycle()
}
}
return nil
}
func (this *EsFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error {
var (
globals = engine.Globals()
pack *engine.PipelinePack
ok = true
count = 0
inChan = r.InChan()
)
LOOP:
for ok {
select {
case pack, ok = <-inChan:
if !ok {
break LOOP
}
if globals.Debug {
globals.Println(*pack)
}
if this.handlePack(pack, h.Project(pack.Project)) {
count += 1
r.Inject(pack)
} else {
pack.Recycle()
}
}
}
globals.Printf("[%s]Total filtered: %d", r.Name(), count)
return nil
}
func (this *EsFilter) Init(config *conf.Conf) {
this.ident = config.String("ident", "")
if this.ident == "" {
panic("empty ident")
}
this.converters = make([]esConverter, 0, 10)
this.indexPattern = config.String("index_pattern", "")
for i := 0; i < len(config.List("converts", nil)); i++ {
section, err := config.Section(fmt.Sprintf("%s[%d]", "converts", i))
if err != nil {
panic(err)
}
c := esConverter{}
c.load(section)
this.converters = append(this.converters, c)
}
geodbFile := config.String("geodbfile", "")
if err := als.LoadGeoDb(geodbFile); err != nil {
panic(err)
}
globals := engine.Globals()
if globals.Verbose {
globals.Printf("Loaded geodb %s\n", geodbFile)
}
}
func (this *CardinalityOutput) handleHttpRequest(w http.ResponseWriter,
req *http.Request, params map[string]interface{}) (interface{}, error) {
vars := mux.Vars(req)
key := vars["key"]
globals := engine.Globals()
if globals.Verbose {
globals.Println(req.Method, key)
}
output := make(map[string]interface{})
switch req.Method {
case "GET":
if key == "all" {
for _, c := range this.counters.Categories() {
output[c] = fmt.Sprintf("[%v] %d",
bjtime.TsToString(int(this.counters.StartedAt(c).Unix())),
this.counters.Count(c))
}
} else {
output[key] = this.counters.Count(key)
}
case "PUT":
this.counters.Reset(key)
output["msg"] = "ok"
}
return output, nil
}
func (this *NetReceiverInput) handleTcpConnection(conn net.Conn,
r engine.InputRunner) {
var (
lineReader = bufio.NewReader(conn)
line string
err error
pack *engine.PipelinePack
ok bool
inChan = r.InChan()
globals = engine.Globals()
)
globals.Printf("Connection from %s", conn.RemoteAddr())
for {
line, err = lineReader.ReadString('\n')
if err != nil {
globals.Printf("[%s]%s", conn.RemoteAddr(), err)
continue
}
atomic.AddInt64(&this.totalBytes, int64(len(line)))
atomic.AddInt64(&this.periodBytes, int64(len(line)))
pack, ok = <-inChan
if !ok {
break
}
// TODO marshal the pack from line
r.Inject(pack)
}
globals.Printf("Closed connection from %s", conn.RemoteAddr().String())
}
func (this *SkyOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
var (
ok = true
pack *engine.PipelinePack
inChan = r.InChan()
globals = engine.Globals()
project = h.Project(this.project)
)
LOOP:
for ok {
select {
case pack, ok = <-inChan:
if !ok {
break LOOP
}
if globals.Debug {
globals.Println(*pack)
}
this.feedSky(project, pack)
pack.Recycle()
}
}
return nil
}
func (this *esBufferWorker) inject(pack *engine.PipelinePack) {
this.timestamp = pack.Message.Timestamp
switch this.expression {
case "count":
this.summary.N += 1
default:
value, err := pack.Message.FieldValue(this.fieldName, this.fieldType)
if err != nil {
globals := engine.Globals()
if globals.Verbose {
globals.Printf("[%s]%v", this.camelName, err)
}
return
}
// add counters
switch this.fieldType {
case als.KEY_TYPE_INT, als.KEY_TYPE_MONEY, als.KEY_TYPE_RANGE:
this.summary.Add(float64(value.(int)))
case als.KEY_TYPE_FLOAT:
this.summary.Add(value.(float64))
}
}
}
func (this *alarmWorker) insert(args ...interface{}) {
if engine.Globals().Debug {
this.project.Printf("%s %+v\n", this.conf.title, args)
}
this.Lock()
this.insertStmt.Exec(args...)
this.Unlock()
}
func (this *NetReceiverInput) reportStats(r engine.InputRunner) {
globals := engine.Globals()
for _ = range r.Ticker() {
globals.Printf("Total %s, speed: %s/s",
gofmt.ByteSize(this.totalBytes),
gofmt.ByteSize(this.periodBytes))
this.periodBytes = int64(0)
}
}
func (this *ArchiveInput) Run(r engine.InputRunner, h engine.PluginHelper) error {
this.runner = r
this.h = h
this.chkpnt.Load()
go func() {
for !this.stopping {
select {
case <-r.Ticker():
this.chkpnt.Dump()
if this.leftN > 0 {
engine.Globals().Printf("[%s]Left %d files", r.Name(), this.leftN)
}
}
}
}()
this.workersWg = new(sync.WaitGroup)
filepath.Walk(this.rootDir, this.setLeftN)
globals := engine.Globals()
if globals.Verbose {
globals.Printf("Total files:%d", this.leftN)
}
// do the real job
filepath.Walk(this.rootDir, this.runSingleLogfile)
// wait for all workers done
this.workersWg.Wait()
this.chkpnt.Dump()
if globals.Verbose {
globals.Printf("[%s]Total msg: %d", r.Name(), this.lineN)
}
return nil
}
// create table schema
// for high TPS, each parser has a dedicated sqlite3 db file
func (this *alarmWorker) createDB() {
const (
DATA_BASEDIR = "data"
SQLITE3_DBFILE_SUFFIX = "sqlite"
)
dsn := fmt.Sprintf("file:%s?cache=shared&mode=rwc",
fmt.Sprintf("%s/%s-%d.%s", DATA_BASEDIR, this.conf.dbName, os.Getpid(),
SQLITE3_DBFILE_SUFFIX))
this.db = sqldb.NewSqlDb(sqldb.DRIVER_SQLITE3, dsn, this.project.Logger)
this.db.SetDebug(engine.Globals().Debug)
this.db.CreateDb(fmt.Sprintf(this.conf.createTable, this.conf.dbName))
}
func (this *SelfSysInput) Run(r engine.InputRunner, h engine.PluginHelper) error {
var (
globals = engine.Globals()
stats = newSysStat()
inChan = r.InChan()
pack *engine.PipelinePack
jsonString string
err error
stopped = false
)
for !stopped {
select {
case <-this.stopChan:
stopped = true
case <-r.Ticker():
// same effect as sleep
}
if stopped {
break
}
stats.gatherStats()
jsonString, err = stats.jsonString()
if err != nil {
globals.Println(err)
continue
}
pack = <-inChan
if err = pack.Message.FromLine(fmt.Sprintf("als,%d,%s",
time.Now().Unix(), jsonString)); err != nil {
globals.Printf("invalid sys stat: %s\n", jsonString)
pack.Recycle()
continue
}
pack.Project = "als"
pack.Ident = this.ident
pack.EsIndex = "fun_als"
pack.EsType = "sys"
r.Inject(pack)
}
return nil
}
func (this *alarmWorker) init(config *conf.Conf, stopChan chan interface{}) {
this.Mutex = new(sync.Mutex)
this.stopChan = stopChan
this.history = make(map[string]int64)
this.conf = alarmWorkerConfig{}
this.conf.init(config)
globals := engine.Globals()
if this.conf.windowSize.Seconds() < 1.0 {
this._instantAlarmOnly = true
if this.conf.beepThreshold > 0 {
globals.Printf("[%s]instant only alarm needn't set 'beep_threshold'",
this.conf.camelName)
}
if this.conf.abnormalBase != 10 {
globals.Printf("[%s]instant only alarm needn't set 'abnormal_base'",
this.conf.camelName)
}
if this.conf.abnormalPercent != 1.5 {
globals.Printf("[%s]instant only alarm needn't set 'abnormal_percent'",
this.conf.camelName)
}
if this.conf.showSummary {
globals.Printf("[%s]instant only alarm needn't set 'show_summary'",
this.conf.camelName)
}
if this.conf.createTable != "" {
globals.Printf("[%s]instant only alarm needn't set 'create_table'",
this.conf.camelName)
}
if this.conf.statsStmt != "" {
globals.Printf("[%s]instant only alarm needn't set 'stats_stmt'",
this.conf.camelName)
}
if this.conf.insertStmt != "" {
globals.Printf("[%s]instant only alarm needn't set 'insert_stmt'",
this.conf.camelName)
}
if this.conf.printFormat != "" {
globals.Printf("[%s]instant only alarm needn't set 'printf'",
this.conf.camelName)
}
if this.conf.dbName != "" {
globals.Printf("[%s]instant only alarm needn't set 'dbname'",
this.conf.camelName)
}
}
}
func (this *EsBufferFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error {
var (
pack *engine.PipelinePack
ok = true
globals = engine.Globals()
inChan = r.InChan()
)
for _, worker := range this.wokers {
go worker.run(r, h)
}
LOOP:
for ok {
select {
case pack, ok = <-inChan:
if !ok {
break LOOP
}
if globals.Debug {
globals.Println(*pack)
}
this.handlePack(pack)
pack.Recycle()
}
}
total := 0
for _, worker := range this.wokers {
total += worker.summary.N
worker.flush(r, h)
}
// all workers will get notified and stop running
close(this.stopChan)
globals.Printf("[%s]Total filtered: %d", r.Name(), total)
return nil
}
func (this *esBufferWorker) flush(r engine.FilterRunner, h engine.PluginHelper) {
if this.summary.N == 0 {
return
}
// generate new pack
pack := h.PipelinePack(0)
switch this.expression {
case "count":
pack.Message.SetField(this.esField, this.summary.N)
case "mean":
pack.Message.SetField(this.esField, this.summary.Mean)
case "max":
pack.Message.SetField(this.esField, this.summary.Max)
case "min":
pack.Message.SetField(this.esField, this.summary.Min)
case "sd":
pack.Message.SetField(this.esField, this.summary.Sd())
case "sum":
pack.Message.SetField(this.esField, this.summary.Sum)
default:
panic("invalid expression: " + this.expression)
}
pack.Message.Timestamp = this.timestamp
pack.Ident = this.ident
pack.EsIndex = indexName(h.Project(this.projectName),
this.indexPattern, time.Unix(int64(this.timestamp), 0))
pack.EsType = this.esType
pack.Project = this.projectName
globals := engine.Globals()
if globals.Debug {
globals.Println(*pack)
}
r.Inject(pack)
this.summary.Reset()
}
func (this *NetReceiverInput) Run(r engine.InputRunner, h engine.PluginHelper) error {
listener, err := net.Listen("tcp4", this.listenAddr)
if err != nil {
panic(err)
}
defer listener.Close()
go this.reportStats(r)
LOOP:
for {
conn, err := listener.Accept()
if err != nil {
engine.Globals().Println(err)
break LOOP
}
go this.handleTcpConnection(conn, r)
}
return nil
}
func (this *EsOutput) showPeriodicalStats() {
if !this.showProgress {
return
}
var (
globals = engine.Globals()
total = 0
)
globals.Printf("ES types: %d, within %s", this.counters.Len(), this.reportInterval)
for _, key := range this.counters.SortedKeys() {
val := this.counters.Get(key)
if val > 0 {
total += val
globals.Printf("%-50s %12s", key, gofmt.Comma(int64(val)))
this.counters.Set(key, 0)
}
}
globals.Printf("%50s %12s", "Sum", gofmt.Comma(int64(total)))
}
// for each inbound pack, this filter will generate several new pack
// the original pack will be recycled immediately
func (this *CardinalityFilter) handlePack(r engine.FilterRunner,
h engine.PluginHelper, pack *engine.PipelinePack) {
globals := engine.Globals()
for _, c := range this.converters {
if !pack.Logfile.MatchPrefix(c.logPrefix) || pack.Project != c.project {
continue
}
for _, f := range c.fields {
val, err := pack.Message.FieldValue(f.key, f.typ)
if err != nil {
if globals.Verbose {
h.Project(c.project).Println(err)
}
return
}
for _, interval := range f.intervals {
// generate new pack
p := h.PipelinePack(pack.MsgLoopCount)
if p == nil {
globals.Println("can't get pack in filter")
continue
}
p.Ident = this.ident
p.Project = c.project
p.CardinalityKey = fmt.Sprintf("%s.%s.%s", pack.Project, f.key, interval)
p.CardinalityData = val
p.CardinalityInterval = interval
r.Inject(p)
}
}
}
}
func (this *EsOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
var (
pack *engine.PipelinePack
reloadChan = make(chan interface{})
ok = true
globals = engine.Globals()
inChan = r.InChan()
reportTicker = time.NewTicker(this.reportInterval)
)
this.indexer = core.NewBulkIndexer(this.bulkMaxConn)
this.indexer.BulkMaxDocs = this.bulkMaxDocs
this.indexer.BulkMaxBuffer = this.bulkMaxBuffer
// start the bulk indexer
this.indexer.Run(this.stopChan)
defer reportTicker.Stop()
observer.Subscribe(engine.RELOAD, reloadChan)
LOOP:
for ok {
select {
case <-this.stopChan:
ok = false
case <-reportTicker.C:
this.showPeriodicalStats()
case <-reloadChan:
// TODO
case <-time.After(this.flushInterval):
this.indexer.Flush()
case pack, ok = <-inChan:
if !ok {
break LOOP
}
if globals.Debug {
globals.Println(*pack)
}
this.feedEs(h.Project(pack.Project), pack)
pack.Recycle()
}
}
engine.Globals().Printf("[%s]Total output to ES: %d", r.Name(), this.totalN)
// before shutdown, flush again
if globals.Verbose {
engine.Globals().Println("Waiting for ES flush...")
}
this.indexer.Flush()
if globals.Verbose {
engine.Globals().Println("ES flushed")
}
// let indexer stop
this.stopChan <- true
return nil
}
func shutdown() {
cleanup()
globals.Printf("Terminated after %s.", time.Since(engine.Globals().StartedAt))
os.Exit(0)
}
func (this *alarmWorker) run(h engine.PluginHelper, goAhead chan bool) {
var (
globals = engine.Globals()
summary = stats.Summary{}
beep bool
ever = true
)
// lazy assignment
this.project = h.Project(this.projName)
if globals.DryRun || this._instantAlarmOnly {
goAhead <- true
return
}
this.createDB()
this.prepareInsertStmt()
this.prepareStatsStmt()
goAhead <- true
for ever {
select {
case <-time.After(this.conf.windowSize):
this.Lock()
windowHead, windowTail, err := this.getWindowBorder()
if err != nil {
this.Unlock()
continue
}
if this.conf.showSummary {
summary.Reset()
}
rows, _ := this.statsStmt.Query(windowTail)
cols, _ := rows.Columns()
colsN := len(cols)
values := make([]interface{}, colsN)
valuePtrs := make([]interface{}, colsN)
rowSeverity := 0
abnormal := false
this.workersMutex.Lock()
this.printWindowTitle(windowHead, windowTail, this.conf.title)
for rows.Next() {
beep = false
for i, _ := range cols {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
// 1st column always being aggregated quantile
var amount = values[0].(int64)
if amount == 0 {
break
}
if this.conf.showSummary {
summary.Add(float64(amount))
}
// beep and feed alarmMail
if this.conf.beepThreshold > 0 && int(amount) >= this.conf.beepThreshold {
beep = true
}
rowSeverity = this.conf.severity * int(amount)
// abnormal change? blink
if this.isAbnormalChange(amount,
this.historyKey(this.conf.printFormat, values)) {
this.blinkColorPrintfLn(this.conf.printFormat, values...)
abnormal = true
// multiply factor
rowSeverity *= this.conf.abnormalSeverityFactor
}
this.colorPrintfLn(beep, this.conf.printFormat, values...)
this.feedAlarmMail(abnormal, rowSeverity, this.conf.printFormat, values...)
}
// show summary
if this.conf.showSummary && summary.N > 0 {
this.colorPrintfLn(false, "Total: %.1f, Mean: %.1f", summary.Sum,
summary.Mean)
}
this.workersMutex.Unlock()
rows.Close()
this.moveWindowForward(windowTail)
this.Unlock()
case <-this.stopChan:
ever = false
}
}
}
func (this *ArchiveInput) doRunSingleLogfile(path string) {
reader := als.NewAlsReader(path)
if e := reader.Open(); e != nil {
panic(e)
}
defer func() {
reader.Close()
this.workersWg.Done()
atomic.AddInt32(&this.leftN, -1)
<-this.workerNChan // release the lock
}()
var (
line []byte
lineN int
inChan = this.runner.InChan()
err error
project = this.h.Project(this.project)
pack *engine.PipelinePack
globals = engine.Globals()
)
for !this.stopping {
line, err = reader.ReadLine()
switch err {
case nil:
lineN += 1
atomic.AddInt64(&this.lineN, 1)
if globals.Verbose && lineN == 1 {
project.Printf("[%s]started\n", path)
}
pack = <-inChan
if err = pack.Message.FromLine(string(line)); err != nil {
if project.ShowError && err != als.ErrEmptyLine {
project.Printf("[%s]%v: %s", path, err, string(line))
}
pack.Recycle()
continue
}
pack.Ident = this.ident
pack.Project = this.project
pack.Logfile.SetPath(path)
if globals.Debug {
globals.Println(*pack)
}
this.runner.Inject(pack)
case io.EOF:
if globals.Verbose {
project.Printf("[%s]done, lines: %d\n", path, lineN)
}
this.chkpnt.Put(path)
this.chkpnt.Dump()
return
default:
// unknown error
panic(err)
}
}
}