func (o *HttpOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) { if or.Encoder() == nil { return errors.New("Encoder must be specified.") } var ( e error outBytes []byte ) inChan := or.InChan() for pack := range inChan { outBytes, e = or.Encode(pack) if e != nil { or.UpdateCursor(pack.QueueCursor) pack.Recycle(fmt.Errorf("can't encode: %s", e)) continue } if outBytes == nil { or.UpdateCursor(pack.QueueCursor) pack.Recycle(nil) continue } if e = o.request(or, outBytes); e != nil { e = pipeline.NewRetryMessageError(e.Error()) pack.Recycle(e) } else { or.UpdateCursor(pack.QueueCursor) pack.Recycle(nil) } } return }
func (o *UdpOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) { if or.Encoder() == nil { return errors.New("Encoder required.") } var ( outBytes []byte e error ) for pack := range or.InChan() { if outBytes, e = or.Encode(pack); e != nil { or.UpdateCursor(pack.QueueCursor) e = fmt.Errorf("Error encoding message: %s", e.Error()) pack.Recycle(e) continue } else if outBytes != nil { msgSize := len(outBytes) if msgSize > o.UdpOutputConfig.MaxMessageSize { or.UpdateCursor(pack.QueueCursor) e = fmt.Errorf("Message has exceeded allowed UDP data size: %d > %d", msgSize, o.UdpOutputConfig.MaxMessageSize) pack.Recycle(e) continue } else { o.conn.Write(outBytes) } } or.UpdateCursor(pack.QueueCursor) pack.Recycle(nil) } return }
func (k *KafkaOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) { defer func() { k.producer.Close() k.client.Close() }() if or.Encoder() == nil { return errors.New("Encoder required.") } inChan := or.InChan() useBuffering := or.UsesBuffering() errChan := k.producer.Errors() var wg sync.WaitGroup wg.Add(1) go k.processKafkaErrors(or, errChan, &wg) var ( pack *pipeline.PipelinePack topic = k.config.Topic key sarama.Encoder ) for pack = range inChan { atomic.AddInt64(&k.processMessageCount, 1) if k.topicVariable != nil { topic = getMessageVariable(pack.Message, k.topicVariable) } if k.hashVariable != nil { key = sarama.StringEncoder(getMessageVariable(pack.Message, k.hashVariable)) } msgBytes, err := or.Encode(pack) if err != nil { atomic.AddInt64(&k.processMessageFailures, 1) or.LogError(err) // Don't retry encoding errors. or.UpdateCursor(pack.QueueCursor) pack.Recycle(nil) continue } if msgBytes == nil { atomic.AddInt64(&k.processMessageDiscards, 1) or.UpdateCursor(pack.QueueCursor) pack.Recycle(nil) continue } err = k.producer.QueueMessage(topic, key, sarama.ByteEncoder(msgBytes)) if err != nil { if !useBuffering { atomic.AddInt64(&k.processMessageFailures, 1) } or.LogError(err) } pack.Recycle(err) } errChan <- Shutdown wg.Wait() return }
func (s *SandboxOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) { var ( pack *pipeline.PipelinePack retval int inChan = or.InChan() duration int64 startTime time.Time ok = true ticker = or.Ticker() ) for ok { select { case pack, ok = <-inChan: if !ok { break } if s.sample { startTime = time.Now() } retval = s.sb.ProcessMessage(pack) if s.sample { duration = time.Since(startTime).Nanoseconds() s.reportLock.Lock() s.processMessageDuration += duration s.processMessageSamples++ s.reportLock.Unlock() } s.sample = 0 == rand.Intn(s.sampleDenominator) or.UpdateCursor(pack.QueueCursor) // TODO: support retries? if retval == 0 { atomic.AddInt64(&s.processMessageCount, 1) pack.Recycle(nil) } else if retval < 0 { atomic.AddInt64(&s.processMessageFailures, 1) var e error em := s.sb.LastError() if len(em) > 0 { e = errors.New(em) } pack.Recycle(e) } else { err = fmt.Errorf("FATAL: %s", s.sb.LastError()) pack.Recycle(err) ok = false } case t := <-ticker: startTime = time.Now() if retval = s.sb.TimerEvent(t.UnixNano()); retval != 0 { err = fmt.Errorf("FATAL: %s", s.sb.LastError()) ok = false } duration = time.Since(startTime).Nanoseconds() s.reportLock.Lock() s.timerEventDuration += duration s.timerEventSamples++ s.reportLock.Unlock() } } if err == nil && s.sbc.TimerEventOnShutdown { if retval = s.sb.TimerEvent(time.Now().UnixNano()); retval != 0 { err = fmt.Errorf("FATAL: %s", s.sb.LastError()) } } destroyErr := s.destroy() if destroyErr != nil { if err != nil { or.LogError(err) } err = destroyErr } return err }
func (k *KafkaOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) { defer func() { k.producer.Close() k.client.Close() }() if or.Encoder() == nil { return errors.New("Encoder required.") } inChan := or.InChan() errChan := k.producer.Errors() pInChan := k.producer.Input() shutdownChan := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) go k.processKafkaErrors(or, errChan, shutdownChan, &wg) var ( pack *pipeline.PipelinePack topic = k.config.Topic key sarama.Encoder ) for pack = range inChan { atomic.AddInt64(&k.processMessageCount, 1) if k.topicVariable != nil { topic = getMessageVariable(pack.Message, k.topicVariable) } if k.hashVariable != nil { key = sarama.StringEncoder(getMessageVariable(pack.Message, k.hashVariable)) } msgBytes, err := or.Encode(pack) if err != nil { atomic.AddInt64(&k.processMessageFailures, 1) or.LogError(err) // Don't retry encoding errors. or.UpdateCursor(pack.QueueCursor) pack.Recycle(nil) continue } if msgBytes == nil { atomic.AddInt64(&k.processMessageDiscards, 1) or.UpdateCursor(pack.QueueCursor) pack.Recycle(nil) continue } pMessage := &sarama.ProducerMessage{ Topic: topic, Key: key, Value: sarama.ByteEncoder(msgBytes), } pInChan <- pMessage pack.Recycle(nil) } close(shutdownChan) wg.Wait() return }