Example #1
0
func (mb *MemoryBuffer) Accept(ctx context.Context, p pipeline.Producer) {
	if pdebug.Enabled {
		g := pdebug.Marker("MemoryBuffer.Accept")
		defer g.End()
	}
	defer func() {
		mb.mutex.Lock()
		close(mb.done)
		mb.mutex.Unlock()
	}()

	for {
		select {
		case <-ctx.Done():
			if pdebug.Enabled {
				pdebug.Printf("MemoryBuffer received context done")
			}
			return
		case v := <-p.OutCh():
			switch v.(type) {
			case error:
				if pipeline.IsEndMark(v.(error)) {
					if pdebug.Enabled {
						pdebug.Printf("MemoryBuffer received end mark (read %d lines)", len(mb.lines))
					}
					return
				}
			case Line:
				mb.mutex.Lock()
				mb.lines = append(mb.lines, v.(Line))
				mb.mutex.Unlock()
			}
		}
	}
}
Example #2
0
func (ecf *ExternalCmdFilter) Accept(ctx context.Context, p pipeline.Producer) {
	if pdebug.Enabled {
		g := pdebug.Marker("ExternalCmdFilter.Accept")
		defer g.End()
	}
	defer ecf.outCh.SendEndMark("end of ExternalCmdFilter")

	buf := make([]Line, 0, ecf.thresholdBufsiz)
	for {
		select {
		case <-ctx.Done():
			if pdebug.Enabled {
				pdebug.Printf("ExternalCmdFilter received done")
			}
			return
		case v := <-p.OutCh():
			switch v.(type) {
			case error:
				if pipeline.IsEndMark(v.(error)) {
					if pdebug.Enabled {
						pdebug.Printf("ExternalCmdFilter received end mark")
					}
					if len(buf) > 0 {
						ecf.launchExternalCmd(ctx, buf)
					}
				}
				return
			case Line:
				if pdebug.Enabled {
					pdebug.Printf("ExternalCmdFilter received new line")
				}
				buf = append(buf, v.(Line))
				if len(buf) < ecf.thresholdBufsiz {
					continue
				}

				ecf.launchExternalCmd(ctx, buf)
				buf = buf[0:0]
			}
		}
	}
}
Example #3
0
func (rf *RegexpFilter) Accept(ctx context.Context, p pipeline.Producer) {
	if pdebug.Enabled {
		g := pdebug.Marker("RegexpFilter.Accept")
		defer g.End()
	}
	defer rf.outCh.SendEndMark("end of RegexpFilter")

	flush := make(chan []Line)
	flushDone := make(chan struct{})
	go func() {
		defer close(flushDone)
		for buf := range flush {
			for _, in := range buf {
				if l, err := rf.filter(in); err == nil {
					rf.outCh.Send(l)
				}
			}
			releaseRegexpFilterBuf(buf)
		}
	}()

	buf := getRegexpFilterBuf()
	defer func() { releaseRegexpFilterBuf(buf) }()
	defer func() { <-flushDone }() // Wait till the flush goroutine is done
	defer close(flush)             // Kill the flush goroutine

	lines := 0
	for {
		select {
		case <-ctx.Done():
			if pdebug.Enabled {
				pdebug.Printf("RegexpFilter received done")
			}
			return
		case v := <-p.OutCh():
			switch v.(type) {
			case error:
				if pipeline.IsEndMark(v.(error)) {
					if pdebug.Enabled {
						pdebug.Printf("RegexpFilter received end mark (read %d lines)", lines+len(buf))
					}
					if len(buf) > 0 {
						flush <- buf
						buf = nil
					}
				}
				return
			case Line:
				if pdebug.Enabled {
					lines++
				}
				// We buffer the lines so that we can receive more lines to
				// process while we filter what we already have. The buffer
				// size is fairly big, because this really only makes a
				// difference if we have a lot of lines to process.
				buf = append(buf, v.(Line))
				if len(buf) >= cap(buf) {
					flush <- buf
					buf = getRegexpFilterBuf()
				}
			}
		}
	}
}