func (this *EsBufferFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error { var ( pack *engine.PipelinePack ok = true globals = engine.Globals() inChan = r.InChan() ) for _, worker := range this.wokers { go worker.run(r, h) } LOOP: for ok { select { case pack, ok = <-inChan: if !ok { break LOOP } if globals.Debug { globals.Println(*pack) } this.handlePack(pack) pack.Recycle() } } total := 0 for _, worker := range this.wokers { total += worker.summary.N worker.flush(r, h) } // all workers will get notified and stop running close(this.stopChan) globals.Printf("[%s]Total filtered: %d", r.Name(), total) return nil }
func (this *EsFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error { var ( globals = engine.Globals() pack *engine.PipelinePack ok = true count = 0 inChan = r.InChan() ) LOOP: for ok { select { case pack, ok = <-inChan: if !ok { break LOOP } if globals.Debug { globals.Println(*pack) } if this.handlePack(pack, h.Project(pack.Project)) { count += 1 r.Inject(pack) } else { pack.Recycle() } } } globals.Printf("[%s]Total filtered: %d", r.Name(), count) return nil }
func (this *esBufferWorker) flush(r engine.FilterRunner, h engine.PluginHelper) { if this.summary.N == 0 { return } // generate new pack pack := h.PipelinePack(0) switch this.expression { case "count": pack.Message.SetField(this.esField, this.summary.N) case "mean": pack.Message.SetField(this.esField, this.summary.Mean) case "max": pack.Message.SetField(this.esField, this.summary.Max) case "min": pack.Message.SetField(this.esField, this.summary.Min) case "sd": pack.Message.SetField(this.esField, this.summary.Sd()) case "sum": pack.Message.SetField(this.esField, this.summary.Sum) default: panic("invalid expression: " + this.expression) } pack.Message.Timestamp = this.timestamp pack.Ident = this.ident pack.EsIndex = indexName(h.Project(this.projectName), this.indexPattern, time.Unix(int64(this.timestamp), 0)) pack.EsType = this.esType pack.Project = this.projectName globals := engine.Globals() if globals.Debug { globals.Println(*pack) } r.Inject(pack) this.summary.Reset() }
func (this *CardinalityFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error { var ( pack *engine.PipelinePack ok = true inChan = r.InChan() ) LOOP: for ok { select { case pack, ok = <-inChan: if !ok { break LOOP } this.handlePack(r, h, pack) pack.Recycle() } } return nil }
// for each inbound pack, this filter will generate several new pack // the original pack will be recycled immediately func (this *CardinalityFilter) handlePack(r engine.FilterRunner, h engine.PluginHelper, pack *engine.PipelinePack) { globals := engine.Globals() for _, c := range this.converters { if !pack.Logfile.MatchPrefix(c.logPrefix) || pack.Project != c.project { continue } for _, f := range c.fields { val, err := pack.Message.FieldValue(f.key, f.typ) if err != nil { if globals.Verbose { h.Project(c.project).Println(err) } return } for _, interval := range f.intervals { // generate new pack p := h.PipelinePack(pack.MsgLoopCount) if p == nil { globals.Println("can't get pack in filter") continue } p.Ident = this.ident p.Project = c.project p.CardinalityKey = fmt.Sprintf("%s.%s.%s", pack.Project, f.key, interval) p.CardinalityData = val p.CardinalityInterval = interval r.Inject(p) } } } }