Exemple #1
0
func (this *EsBufferFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error {
	var (
		pack    *engine.PipelinePack
		ok      = true
		globals = engine.Globals()
		inChan  = r.InChan()
	)

	for _, worker := range this.wokers {
		go worker.run(r, h)
	}

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			if globals.Debug {
				globals.Println(*pack)
			}

			this.handlePack(pack)
			pack.Recycle()
		}
	}

	total := 0
	for _, worker := range this.wokers {
		total += worker.summary.N
		worker.flush(r, h)
	}

	// all workers will get notified and stop running
	close(this.stopChan)

	globals.Printf("[%s]Total filtered: %d", r.Name(), total)

	return nil
}
Exemple #2
0
func (this *EsFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error {
	var (
		globals = engine.Globals()
		pack    *engine.PipelinePack
		ok      = true
		count   = 0
		inChan  = r.InChan()
	)

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			if globals.Debug {
				globals.Println(*pack)
			}

			if this.handlePack(pack, h.Project(pack.Project)) {
				count += 1
				r.Inject(pack)
			} else {
				pack.Recycle()
			}
		}
	}

	globals.Printf("[%s]Total filtered: %d", r.Name(), count)

	return nil
}
Exemple #3
0
func (this *esBufferWorker) flush(r engine.FilterRunner, h engine.PluginHelper) {
	if this.summary.N == 0 {
		return
	}

	// generate new pack
	pack := h.PipelinePack(0)

	switch this.expression {
	case "count":
		pack.Message.SetField(this.esField, this.summary.N)
	case "mean":
		pack.Message.SetField(this.esField, this.summary.Mean)
	case "max":
		pack.Message.SetField(this.esField, this.summary.Max)
	case "min":
		pack.Message.SetField(this.esField, this.summary.Min)
	case "sd":
		pack.Message.SetField(this.esField, this.summary.Sd())
	case "sum":
		pack.Message.SetField(this.esField, this.summary.Sum)
	default:
		panic("invalid expression: " + this.expression)
	}

	pack.Message.Timestamp = this.timestamp
	pack.Ident = this.ident
	pack.EsIndex = indexName(h.Project(this.projectName),
		this.indexPattern, time.Unix(int64(this.timestamp), 0))
	pack.EsType = this.esType
	pack.Project = this.projectName
	globals := engine.Globals()
	if globals.Debug {
		globals.Println(*pack)
	}
	r.Inject(pack)

	this.summary.Reset()
}
Exemple #4
0
func (this *CardinalityFilter) Run(r engine.FilterRunner,
	h engine.PluginHelper) error {
	var (
		pack   *engine.PipelinePack
		ok     = true
		inChan = r.InChan()
	)

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			this.handlePack(r, h, pack)
			pack.Recycle()
		}
	}

	return nil
}
Exemple #5
0
// for each inbound pack, this filter will generate several new pack
// the original pack will be recycled immediately
func (this *CardinalityFilter) handlePack(r engine.FilterRunner,
	h engine.PluginHelper, pack *engine.PipelinePack) {
	globals := engine.Globals()
	for _, c := range this.converters {
		if !pack.Logfile.MatchPrefix(c.logPrefix) || pack.Project != c.project {
			continue
		}

		for _, f := range c.fields {
			val, err := pack.Message.FieldValue(f.key, f.typ)
			if err != nil {
				if globals.Verbose {
					h.Project(c.project).Println(err)
				}

				return
			}

			for _, interval := range f.intervals {
				// generate new pack
				p := h.PipelinePack(pack.MsgLoopCount)
				if p == nil {
					globals.Println("can't get pack in filter")
					continue
				}

				p.Ident = this.ident
				p.Project = c.project
				p.CardinalityKey = fmt.Sprintf("%s.%s.%s", pack.Project, f.key, interval)
				p.CardinalityData = val
				p.CardinalityInterval = interval

				r.Inject(p)
			}
		}
	}
}