// Framed protobuf message parser
func (lsi *LogstreamInput) messageProtoParser(ir p.InputRunner, deliver Deliver, stop chan chan bool) (err error) {
var (
pack *p.PipelinePack
record []byte
n int
)
for err == nil {
select {
case lsi.stopped = <-stop:
return
default:
}
n, record, err = lsi.parser.Parse(lsi.stream)
if n > 0 {
lsi.stream.FlushBuffer(n)
}
if len(record) > 0 {
pack = <-ir.InChan()
headerLen := int(record[1]) + 3 // recsep+len+header+unitsep
messageLen := len(record) - headerLen
// ignore authentication headers
if messageLen > cap(pack.MsgBytes) {
pack.MsgBytes = make([]byte, messageLen)
}
pack.MsgBytes = pack.MsgBytes[:messageLen]
copy(pack.MsgBytes, record[headerLen:])
deliver(pack)
lsi.countRecord()
}
}
return
}
func (zi *ZeroMQInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error {
// Get the InputRunner's chan to receive empty PipelinePacks
packs := ir.InChan()
var decoding chan<- *pipeline.PipelinePack
if zi.conf.Decoder != "" {
// Fetch specified decoder
decoder, ok := h.DecoderSet().ByName(zi.conf.Decoder)
if !ok {
err := fmt.Errorf("Could not find decoder", zi.conf.Decoder)
return err
}
// Get the decoder's receiving chan
decoding = decoder.InChan()
}
var pack *pipeline.PipelinePack
var count int
var b []byte
var err error
// Read data from websocket broadcast chan
for {
b, err = zi.socket.Recv(0)
if err != nil {
ir.LogError(err)
continue
}
// Grab an empty PipelinePack from the InputRunner
pack = <-packs
// Trim the excess empty bytes
count = len(b)
pack.MsgBytes = pack.MsgBytes[:count]
// Copy ws bytes into pack's bytes
copy(pack.MsgBytes, b)
if decoding != nil {
// Send pack onto decoder
decoding <- pack
} else {
// Send pack into Heka pipeline
ir.Inject(pack)
}
}
return nil
}
func (s *SandboxEncoder) Encode(pack *pipeline.PipelinePack) (output []byte, err error) {
if s.sb == nil {
err = errors.New("No sandbox.")
return
}
atomic.AddInt64(&s.processMessageCount, 1)
s.injected = false
var startTime time.Time
if s.sample {
startTime = time.Now()
}
cowpack := new(pipeline.PipelinePack)
cowpack.Message = pack.Message // the actual copy will happen if write_message is called
cowpack.MsgBytes = pack.MsgBytes // no copying is necessary since we don't change it
retval := s.sb.ProcessMessage(cowpack)
if retval == 0 && !s.injected {
// `inject_message` was never called, protobuf encode the copy on write
// message.
if s.output, err = s.cEncoder.EncodeMessage(cowpack.Message); err != nil {
return
}
}
if s.sample {
duration := time.Since(startTime).Nanoseconds()
s.reportLock.Lock()
s.processMessageDuration += duration
s.processMessageSamples++
s.reportLock.Unlock()
}
s.sample = 0 == rand.Intn(s.sampleDenominator)
if retval > 0 {
err = fmt.Errorf("FATAL: %s", s.sb.LastError())
return
}
if retval == -2 {
// Encoder has nothing to return.
return nil, nil
}
if retval < 0 {
atomic.AddInt64(&s.processMessageFailures, 1)
err = fmt.Errorf("Failed serializing: %s", s.sb.LastError())
return
}
return s.output, nil
}
func (ri *RedisMQInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error {
// Get the InputRunner's chan to receive empty PipelinePacks
packs := ir.InChan()
var decoding chan<- *pipeline.PipelinePack
if ri.conf.Decoder != "" {
// Fetch specified decoder
decoder, ok := h.DecoderRunner(ri.conf.Decoder)
if !ok {
err := fmt.Errorf("Could not find decoder", ri.conf.Decoder)
return err
}
// Get the decoder's receiving chan
decoding = decoder.InChan()
}
var pack *pipeline.PipelinePack
//var p []*redismq.Package
var p *redismq.Package
var count int
var b []byte
var err error
for {
p, err = ri.rdconsumer.Get()
if err != nil {
ir.LogError(err)
continue
}
err = p.Ack()
if err != nil {
ir.LogError(err)
}
b = []byte(p.Payload)
// Grab an empty PipelinePack from the InputRunner
pack = <-packs
// Trim the excess empty bytes
count = len(b)
pack.MsgBytes = pack.MsgBytes[:count]
// Copy ws bytes into pack's bytes
copy(pack.MsgBytes, b)
if decoding != nil {
// Send pack onto decoder
decoding <- pack
} else {
// Send pack into Heka pipeline
ir.Inject(pack)
}
}
/*
checkStat := time.Tick(ri.statInterval)
ok := true
for ok {
select {
case _, ok = <-ri.stopChan:
break
case <-checkStat:
p, err = ri.rdconsumer.MultiGet(500)
if err != nil {
ir.LogError(err)
continue
}
err = p[len(p)-1].MultiAck()
if err != nil {
ir.LogError(err)
}
for _, v := range p {
b = []byte(v.Payload)
// Grab an empty PipelinePack from the InputRunner
pack = <-packs
// Trim the excess empty bytes
count = len(b)
pack.MsgBytes = pack.MsgBytes[:count]
// Copy ws bytes into pack's bytes
copy(pack.MsgBytes, b)
if decoding != nil {
// Send pack onto decoder
decoding <- pack
} else {
// Send pack into Heka pipeline
ir.Inject(pack)
}
}
}
}
*/
return nil
}
