// Framed protobuf message parser func (lsi *LogstreamInput) messageProtoParser(ir p.InputRunner, deliver Deliver, stop chan chan bool) (err error) { var ( pack *p.PipelinePack record []byte n int ) for err == nil { select { case lsi.stopped = <-stop: return default: } n, record, err = lsi.parser.Parse(lsi.stream) if n > 0 { lsi.stream.FlushBuffer(n) } if len(record) > 0 { pack = <-ir.InChan() headerLen := int(record[1]) + 3 // recsep+len+header+unitsep messageLen := len(record) - headerLen // ignore authentication headers if messageLen > cap(pack.MsgBytes) { pack.MsgBytes = make([]byte, messageLen) } pack.MsgBytes = pack.MsgBytes[:messageLen] copy(pack.MsgBytes, record[headerLen:]) deliver(pack) lsi.countRecord() } } return }
func (zi *ZeroMQInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error { // Get the InputRunner's chan to receive empty PipelinePacks packs := ir.InChan() var decoding chan<- *pipeline.PipelinePack if zi.conf.Decoder != "" { // Fetch specified decoder decoder, ok := h.DecoderSet().ByName(zi.conf.Decoder) if !ok { err := fmt.Errorf("Could not find decoder", zi.conf.Decoder) return err } // Get the decoder's receiving chan decoding = decoder.InChan() } var pack *pipeline.PipelinePack var count int var b []byte var err error // Read data from websocket broadcast chan for { b, err = zi.socket.Recv(0) if err != nil { ir.LogError(err) continue } // Grab an empty PipelinePack from the InputRunner pack = <-packs // Trim the excess empty bytes count = len(b) pack.MsgBytes = pack.MsgBytes[:count] // Copy ws bytes into pack's bytes copy(pack.MsgBytes, b) if decoding != nil { // Send pack onto decoder decoding <- pack } else { // Send pack into Heka pipeline ir.Inject(pack) } } return nil }
func (s *SandboxEncoder) Encode(pack *pipeline.PipelinePack) (output []byte, err error) { if s.sb == nil { err = errors.New("No sandbox.") return } atomic.AddInt64(&s.processMessageCount, 1) s.injected = false var startTime time.Time if s.sample { startTime = time.Now() } cowpack := new(pipeline.PipelinePack) cowpack.Message = pack.Message // the actual copy will happen if write_message is called cowpack.MsgBytes = pack.MsgBytes // no copying is necessary since we don't change it retval := s.sb.ProcessMessage(cowpack) if retval == 0 && !s.injected { // `inject_message` was never called, protobuf encode the copy on write // message. if s.output, err = s.cEncoder.EncodeMessage(cowpack.Message); err != nil { return } } if s.sample { duration := time.Since(startTime).Nanoseconds() s.reportLock.Lock() s.processMessageDuration += duration s.processMessageSamples++ s.reportLock.Unlock() } s.sample = 0 == rand.Intn(s.sampleDenominator) if retval > 0 { err = fmt.Errorf("FATAL: %s", s.sb.LastError()) return } if retval == -2 { // Encoder has nothing to return. return nil, nil } if retval < 0 { atomic.AddInt64(&s.processMessageFailures, 1) err = fmt.Errorf("Failed serializing: %s", s.sb.LastError()) return } return s.output, nil }
func (ri *RedisMQInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error { // Get the InputRunner's chan to receive empty PipelinePacks packs := ir.InChan() var decoding chan<- *pipeline.PipelinePack if ri.conf.Decoder != "" { // Fetch specified decoder decoder, ok := h.DecoderRunner(ri.conf.Decoder) if !ok { err := fmt.Errorf("Could not find decoder", ri.conf.Decoder) return err } // Get the decoder's receiving chan decoding = decoder.InChan() } var pack *pipeline.PipelinePack //var p []*redismq.Package var p *redismq.Package var count int var b []byte var err error for { p, err = ri.rdconsumer.Get() if err != nil { ir.LogError(err) continue } err = p.Ack() if err != nil { ir.LogError(err) } b = []byte(p.Payload) // Grab an empty PipelinePack from the InputRunner pack = <-packs // Trim the excess empty bytes count = len(b) pack.MsgBytes = pack.MsgBytes[:count] // Copy ws bytes into pack's bytes copy(pack.MsgBytes, b) if decoding != nil { // Send pack onto decoder decoding <- pack } else { // Send pack into Heka pipeline ir.Inject(pack) } } /* checkStat := time.Tick(ri.statInterval) ok := true for ok { select { case _, ok = <-ri.stopChan: break case <-checkStat: p, err = ri.rdconsumer.MultiGet(500) if err != nil { ir.LogError(err) continue } err = p[len(p)-1].MultiAck() if err != nil { ir.LogError(err) } for _, v := range p { b = []byte(v.Payload) // Grab an empty PipelinePack from the InputRunner pack = <-packs // Trim the excess empty bytes count = len(b) pack.MsgBytes = pack.MsgBytes[:count] // Copy ws bytes into pack's bytes copy(pack.MsgBytes, b) if decoding != nil { // Send pack onto decoder decoding <- pack } else { // Send pack into Heka pipeline ir.Inject(pack) } } } } */ return nil }