// Run is the plugin's main loop
//iterates over received messages, checking against
//message hostname and delivering to the output if hostname is in our config.
func (o *EmailOutput) Run(runner pipeline.OutputRunner, helper pipeline.PluginHelper) (
err error) {
var (
payload string
)
body := bytes.NewBuffer(nil)
for pack := range runner.InChan() {
payload = pack.Message.GetPayload()
if len(payload) > 100 {
payload = payload[:100]
}
body.WriteString(fmt.Sprintf("Subject: %s [%d] %s@%s: ",
utils.TsTime(pack.Message.GetTimestamp()).Format(time.RFC3339),
pack.Message.GetSeverity(), pack.Message.GetLogger(),
pack.Message.GetHostname()))
body.WriteString(payload)
body.WriteString("\r\n\r\n")
body.WriteString(pack.Message.GetPayload())
pack.Recycle()
err = o.sendMail(body.Bytes())
body.Reset()
if err != nil {
return fmt.Errorf("error sending email: %s", err)
}
}
return
}
func (o *HttpOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
if or.Encoder() == nil {
return errors.New("Encoder must be specified.")
}
var (
e error
outBytes []byte
)
inChan := or.InChan()
for pack := range inChan {
outBytes, e = or.Encode(pack)
pack.Recycle()
if e != nil {
or.LogError(e)
continue
}
if outBytes == nil {
continue
}
if e = o.request(or, outBytes); e != nil {
or.LogError(e)
}
}
return
}
func (k *KafkaOutput) processKafkaErrors(or pipeline.OutputRunner, errChan <-chan *sarama.ProducerError,
shutdownChan chan struct{}, wg *sync.WaitGroup) {
var (
ok = true
pErr *sarama.ProducerError
)
for ok {
select {
case pErr, ok = <-errChan:
if !ok {
break
}
err := pErr.Err
switch err.(type) {
case sarama.PacketEncodingError:
atomic.AddInt64(&k.kafkaEncodingErrors, 1)
or.LogError(fmt.Errorf("kafka encoding error: %s", err.Error()))
default:
atomic.AddInt64(&k.kafkaDroppedMessages, 1)
if err != nil {
msgValue, _ := pErr.Msg.Value.Encode()
or.LogError(fmt.Errorf("kafka error '%s' for message '%s'", err.Error(),
string(msgValue)))
}
}
case <-shutdownChan:
ok = false
break
}
}
wg.Done()
}
func (o *UdpOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
if or.Encoder() == nil {
return errors.New("Encoder required.")
}
var (
outBytes []byte
e error
)
for pack := range or.InChan() {
if outBytes, e = or.Encode(pack); e != nil {
or.LogError(fmt.Errorf("Error encoding message: %s", e.Error()))
} else if outBytes != nil {
msgSize := len(outBytes)
if msgSize > o.UdpOutputConfig.MaxMessageSize {
or.LogError(fmt.Errorf("Message has exceeded allowed UDP data size: %d > %d", msgSize, o.UdpOutputConfig.MaxMessageSize))
} else {
o.conn.Write(outBytes)
}
}
pack.Recycle()
}
return
}
func (wso *WebSocketsOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) error {
for pc := range or.InChan() {
wso.broadcast <- pc.Pack.Message
pc.Pack.Recycle()
}
return nil
}
func (clo *CloudLoggingOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
var (
pack *pipeline.PipelinePack
e error
k string
m *logging.LogEntry
exist bool
ok = true
inChan = or.InChan()
groupBatch = make(map[string]*LogBatch)
outBatch *LogBatch
ticker = time.Tick(time.Duration(clo.conf.FlushInterval) * time.Millisecond)
)
clo.or = or
go clo.committer()
for ok {
select {
case pack, ok = <-inChan:
// Closed inChan => we're shutting down, flush data.
if !ok {
clo.sendGroupBatch(groupBatch)
close(clo.batchChan)
<-clo.outputExit
break
}
k, m, e = clo.Encode(pack)
pack.Recycle()
if e != nil {
or.LogError(e)
continue
}
if k != "" && m != nil {
outBatch, exist = groupBatch[k]
if !exist {
outBatch = &LogBatch{count: 0, batch: make([]*logging.LogEntry, 0, 100), name: k}
groupBatch[k] = outBatch
}
outBatch.batch = append(outBatch.batch, m)
if outBatch.count++; clo.CheckFlush(int(outBatch.count), len(outBatch.batch)) {
if len(outBatch.batch) > 0 {
outBatch.batch = clo.sendBatch(k, outBatch.batch, outBatch.count)
outBatch.count = 0
}
}
}
case <-ticker:
clo.sendGroupBatch(groupBatch)
case err = <-clo.outputExit:
ok = false
}
}
return
}
func (ro *RedisMQOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) error {
var outgoing string
for pack := range or.InChan() {
outgoing = fmt.Sprintf("%s", pack.Message.GetPayload())
ro.rdqueue.Put(outgoing)
pack.Recycle()
}
return nil
}
func (cmo *CloudMonitoringOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
var (
pack *pipeline.PipelinePack
e error
m *cloudmonitoring.TimeseriesPoint
ok = true
count int64
inChan = or.InChan()
outBatch = make([]*cloudmonitoring.TimeseriesPoint, 0, 200)
ticker = time.Tick(time.Duration(cmo.conf.FlushInterval) * time.Millisecond)
)
cmo.or = or
go cmo.committer()
for ok {
select {
case pack, ok = <-inChan:
// Closed inChan => we're shutting down, flush data.
if !ok {
if len(outBatch) > 0 {
cmo.sendBatch(outBatch, count)
}
close(cmo.batchChan)
<-cmo.outputExit
break
}
m, e = cmo.Encode(pack)
pack.Recycle()
if e != nil {
or.LogError(e)
continue
}
if m != nil {
outBatch = append(outBatch, m)
if count++; cmo.CheckFlush(int(count), len(outBatch)) {
if len(outBatch) > 0 {
outBatch = cmo.sendBatch(outBatch, count)
count = 0
}
}
}
case <-ticker:
if len(outBatch) > 0 {
outBatch = cmo.sendBatch(outBatch, count)
}
count = 0
case err = <-cmo.outputExit:
ok = false
}
}
return
}
func (o *UdpOutput) Run(runner pipeline.OutputRunner, helper pipeline.PluginHelper) (
err error) {
var outgoing string
for pack := range runner.InChan() {
outgoing = fmt.Sprintf("%s\n", pack.Message.GetPayload())
o.conn.Write([]byte(outgoing))
pack.Recycle()
}
return
}
func (cef *CefOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
var (
facility, priority syslog.Priority
ident string
ok bool
p syslog.Priority
e error
pack *pipeline.PipelinePack
)
syslogMsg := new(SyslogMsg)
for pack = range or.InChan() {
// default values
facility, priority = syslog.LOG_LOCAL4, syslog.LOG_INFO
ident = "heka_no_ident"
priField := pack.Message.FindFirstField("cef_meta.syslog_priority")
if priField != nil {
priStr := priField.ValueString[0]
if p, ok = SYSLOG_PRIORITY[priStr]; ok {
priority = p
}
}
facField := pack.Message.FindFirstField("cef_meta.syslog_facility")
if facField != nil {
facStr := facField.ValueString[0]
if p, ok = SYSLOG_FACILITY[facStr]; ok {
facility = p
}
}
idField := pack.Message.FindFirstField("cef_meta.syslog_ident")
if idField != nil {
ident = idField.ValueString[0]
}
syslogMsg.priority = priority | facility
syslogMsg.prefix = ident
syslogMsg.payload = pack.Message.GetPayload()
pack.Recycle()
_, e = cef.syslogWriter.WriteString(syslogMsg.priority, syslogMsg.prefix,
syslogMsg.payload)
if e != nil {
or.LogError(e)
}
}
cef.syslogWriter.Close()
return
}
func (k *KafkaOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
defer func() {
k.producer.Close()
k.client.Close()
}()
if or.Encoder() == nil {
return errors.New("Encoder required.")
}
inChan := or.InChan()
errChan := k.producer.Errors()
var wg sync.WaitGroup
wg.Add(1)
go k.processKafkaErrors(or, errChan, &wg)
var (
pack *pipeline.PipelinePack
topic = k.config.Topic
key sarama.Encoder
)
for pack = range inChan {
atomic.AddInt64(&k.processMessageCount, 1)
if k.topicVariable != nil {
topic = getMessageVariable(pack.Message, k.topicVariable)
}
if k.hashVariable != nil {
key = sarama.StringEncoder(getMessageVariable(pack.Message, k.hashVariable))
}
if msgBytes, err := or.Encode(pack); err == nil {
if msgBytes != nil {
err = k.producer.QueueMessage(topic, key, sarama.ByteEncoder(msgBytes))
if err != nil {
atomic.AddInt64(&k.processMessageFailures, 1)
or.LogError(err)
}
} else {
atomic.AddInt64(&k.processMessageDiscards, 1)
}
} else {
atomic.AddInt64(&k.processMessageFailures, 1)
or.LogError(err)
}
pack.Recycle()
}
errChan <- Shutdown
wg.Wait()
return
}
func (rop *RedisOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) error {
inChan := or.InChan()
for pack := range inChan {
payload := pack.Message.GetPayload()
_, err := rop.conn.Do("LPUSH", rop.conf.Key, payload)
if err != nil {
or.LogError(err)
continue
}
pack.Recycle()
}
return nil
}
func (rlo *RedisListOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) error {
inChan := or.InChan()
for pack := range inChan {
payload := pack.Message.GetPayload()
_, err := rlo.conn.Do("LPUSH", rlo.conf.ListName, payload)
if err != nil {
or.LogError(fmt.Errorf("Redis LPUSH error: %s", err))
continue
}
pack.Recycle(nil)
}
return nil
}
func (f *FirehoseOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) error {
for pack := range or.InChan() {
payload := pack.Message.GetPayload()
timestamp := time.Unix(0, pack.Message.GetTimestamp()).Format("2006-01-02 15:04:05.000")
pack.Recycle(nil)
// Verify input is valid json
object := make(map[string]interface{})
err := json.Unmarshal([]byte(payload), &object)
if err != nil {
or.LogError(err)
continue
}
if f.timestampColumn != "" {
// add Heka message's timestamp to column named in timestampColumn
object[f.timestampColumn] = timestamp
}
record, err := json.Marshal(object)
if err != nil {
or.LogError(err)
continue
}
// Send data to the firehose
err = f.client.PutRecord(record)
if err != nil {
or.LogError(err)
continue
}
}
return nil
}
func (k *KinesisOutput) SendEntries(or pipeline.OutputRunner, entries []*kin.PutRecordsRequestEntry, backoff time.Duration, retries int) error {
k.hasTriedToSend = true
multParams := &kin.PutRecordsInput{
Records: entries,
StreamName: aws.String(k.config.Stream),
}
data, err := k.Client.PutRecords(multParams)
// Update statistics & handle errors
if err != nil {
if or != nil {
or.LogError(fmt.Errorf("Batch: Error pushing message to Kinesis: %s", err))
}
atomic.AddInt64(&k.batchesFailed, 1)
if retries <= k.config.MaxRetries || k.config.MaxRetries == -1 {
atomic.AddInt64(&k.retryCount, 1)
time.Sleep(backoff + k.backoffIncrement)
// filter down to only the failed records:
retryEntries := []*kin.PutRecordsRequestEntry{}
for i, entry := range entries {
response := data.Records[i]
if response.ErrorCode != nil {
// incase we are rate limited push the entry to a new shard.
entry.PartitionKey = aws.String(fmt.Sprintf("%d", rand.Int63()))
retryEntries = append(retryEntries, entry)
}
}
k.SendEntries(or, retryEntries, backoff+k.backoffIncrement, retries+1)
} else {
atomic.AddInt64(&k.dropMessageCount, int64(len(entries)))
if or != nil {
or.LogError(fmt.Errorf("Batch: Hit max retries when attempting to send data"))
}
}
}
atomic.AddInt64(&k.batchesSent, 1)
return nil
}
func (k *KinesisOutput) HandlePackage(or pipeline.OutputRunner, pack *pipeline.PipelinePack) error {
// If we are flushing, wait until we have finished.
k.flushLock.Lock()
defer k.flushLock.Unlock()
// encode the packages.
msg, err := or.Encode(pack)
if err != nil {
errOut := fmt.Errorf("Error encoding message: %s", err)
or.LogError(errOut)
pack.Recycle(nil)
return errOut
}
// If we only care about the Payload...
if k.config.PayloadOnly {
msg = []byte(pack.Message.GetPayload())
}
var tmp []byte
// if we already have data then we should append.
if len(k.batchedData) > 0 {
tmp = append(append(k.batchedData, []byte(",")...), msg...)
} else {
tmp = msg
}
// if we can't fit the data in this record
if len(tmp) > k.KINESIS_RECORD_SIZE {
// add the existing data to the output batch
array := append(append([]byte("["), k.batchedData...), []byte("]")...)
k.AddToRecordBatch(or, array)
// update the batched data to only contain the current message.
k.batchedData = msg
} else {
// otherwise we add the existing data to a batch
k.batchedData = tmp
}
// do reporting and tidy up
atomic.AddInt64(&k.processMessageCount, 1)
pack.Recycle(nil)
return nil
}
func (so *SentryOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
var (
udpAddrStr string
udpAddr *net.UDPAddr
socket net.Conn
e error
ok bool
pack *pipeline.PipelinePack
)
sentryMsg := &SentryMsg{
dataPacket: make([]byte, 0, so.config.MaxSentryBytes),
}
for pack = range or.InChan() {
e = so.prepSentryMsg(pack, sentryMsg)
pack.Recycle()
if e != nil {
or.LogError(e)
continue
}
udpAddrStr = sentryMsg.parsedDsn.Host
if socket, ok = so.udpMap[udpAddrStr]; !ok {
if len(so.udpMap) > so.config.MaxUdpSockets {
or.LogError(fmt.Errorf("Max # of UDP sockets [%d] reached.",
so.config.MaxUdpSockets))
continue
}
if udpAddr, e = net.ResolveUDPAddr("udp", udpAddrStr); e != nil {
or.LogError(fmt.Errorf("can't resolve UDP address %s: %s",
udpAddrStr, e))
continue
}
if socket, e = net.DialUDP("udp", nil, udpAddr); e != nil {
or.LogError(fmt.Errorf("can't dial UDP socket: %s", e))
continue
}
so.udpMap[sentryMsg.parsedDsn.Host] = socket
}
socket.Write(sentryMsg.dataPacket)
}
return
}
func (zo *ZeroMQOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) error {
defer func() {
zo.socket.Close()
zo.context.Close()
}()
var b []byte
var p [][]byte
for pc := range or.InChan() {
b = pc.Pack.MsgBytes
p = [][]byte{nil, b}
zo.socket.SendMultipart(p, 0)
pc.Pack.Recycle()
}
return nil
}
func (k *KafkaOutput) processKafkaErrors(or pipeline.OutputRunner, errChan chan error, wg *sync.WaitGroup) {
shutdown:
for err := range errChan {
switch err {
case nil:
case Shutdown:
break shutdown
case sarama.EncodingError:
atomic.AddInt64(&k.kafkaEncodingErrors, 1)
default:
if e, ok := err.(sarama.DroppedMessagesError); ok {
atomic.AddInt64(&k.kafkaDroppedMessages, int64(e.DroppedMessages))
}
or.LogError(err)
}
}
wg.Done()
}
func (cwo *CloudwatchOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
inChan := or.InChan()
payloads := make(chan CloudwatchDatapoints, cwo.backlog)
go cwo.Submitter(payloads, or)
var (
pack *pipeline.PipelinePack
msg *message.Message
rawDataPoints *CloudwatchDatapointPayload
dataPoints *CloudwatchDatapoints
)
dataPoints = new(CloudwatchDatapoints)
dataPoints.Datapoints = make([]cloudwatch.MetricDatum, 0, 0)
for pack = range inChan {
rawDataPoints = new(CloudwatchDatapointPayload)
msg = pack.Message
err = json.Unmarshal([]byte(msg.GetPayload()), rawDataPoints)
if err != nil {
or.LogMessage(fmt.Sprintf("warning, unable to parse payload: %s", err))
err = nil
continue
}
// Run through the list and convert them to CloudwatchDatapoints
for _, rawDatum := range rawDataPoints.Datapoints {
datum := cloudwatch.MetricDatum{
Dimensions: rawDatum.Dimensions,
MetricName: rawDatum.MetricName,
Unit: rawDatum.Unit,
Value: rawDatum.Value,
StatisticValues: rawDatum.StatisticValues,
}
if rawDatum.Timestamp != "" {
parsedTime, err := message.ForgivingTimeParse("", rawDatum.Timestamp, cwo.tzLocation)
if err != nil {
or.LogMessage(fmt.Sprintf("unable to parse timestamp for datum: %s", rawDatum))
continue
}
datum.Timestamp = parsedTime
}
dataPoints.Datapoints = append(dataPoints.Datapoints, datum)
}
payloads <- *dataPoints
dataPoints.Datapoints = dataPoints.Datapoints[:0]
rawDataPoints.Datapoints = rawDataPoints.Datapoints[:0]
pack.Recycle()
}
or.LogMessage("shutting down AWS Cloudwatch submitter")
cwo.stopChan <- true
<-cwo.stopChan
return
}
func (k *KinesisOutput) Run(or pipeline.OutputRunner, helper pipeline.PluginHelper) error {
var (
pack *pipeline.PipelinePack
msg []byte
pk string
err error
params *kin.PutRecordInput
)
if or.Encoder() == nil {
return fmt.Errorf("Encoder required.")
}
for pack = range or.InChan() {
msg, err = or.Encode(pack)
if err != nil {
or.LogError(fmt.Errorf("Error encoding message: %s", err))
pack.Recycle(nil)
continue
}
pk = fmt.Sprintf("%d-%s", pack.Message.Timestamp, pack.Message.Hostname)
if k.config.PayloadOnly {
msg = []byte(pack.Message.GetPayload())
}
params = &kin.PutRecordInput{
Data: msg,
PartitionKey: aws.String(pk),
StreamName: aws.String(k.config.Stream),
}
_, err = k.Client.PutRecord(params)
if err != nil {
or.LogError(fmt.Errorf("Error pushing message to Kinesis: %s", err))
pack.Recycle(nil)
continue
}
pack.Recycle(nil)
}
return nil
}
func (sop *SCAMPOutputPlugin) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
var pack *pipeline.PipelinePack
// We have no default encoder
if or.Encoder() == nil {
return errors.New("Encoder required.")
}
for pack = range or.InChan() {
scamp.Info.Printf("received pipeline pack")
encoded, err := or.Encode(pack) // pack.Message.GetPayload()
if err == nil {
scamp.Info.Printf("payload: %s", encoded)
msg := scamp.NewMessage()
msg.SetEnvelope(scamp.ENVELOPE_JSON)
msg.SetAction(sop.conf.Action)
msg.SetVersion(1)
msg.Write(encoded)
}
pack.Recycle(err)
}
fmt.Println("sup from end of for loop in Run")
return
}
// Run is the plugin's main loop
//iterates over received messages, checking against
//message hostname and delivering to the output if hostname is in our config.
func (o *MantisOutput) Run(runner pipeline.OutputRunner, helper pipeline.PluginHelper) (
err error) {
var (
short, long string
//issue int
)
for pack := range runner.InChan() {
long = pack.Message.GetPayload()
short = fmt.Sprintf("%s [%d] %s@%s: %s",
utils.TsTime(pack.Message.GetTimestamp()).Format(time.RFC3339),
pack.Message.GetSeverity(), pack.Message.GetLogger(),
pack.Message.GetHostname(), long)
pack.Recycle()
if _, err = o.sender.Send(short, long); err != nil {
return fmt.Errorf("error sending to %s: %s", o.sender.URL, err)
}
}
return
}
// Fetch correct output and iterate over received messages, checking against
// message hostname and delivering to the output if hostname is in our config.
func (f *HostFilter) Run(runner pipeline.FilterRunner, helper pipeline.PluginHelper) (
err error) {
var (
hostname string
output pipeline.OutputRunner
ok bool
)
if output, ok = helper.Output(f.output); !ok {
return fmt.Errorf("No output: %s", output)
}
for pack := range runner.InChan() {
hostname = pack.Message.GetHostname()
if f.hosts[hostname] {
output.InChan() <- pack
} else {
pack.Recycle()
}
}
return
}
func (output *IrcOutput) Run(runner pipeline.OutputRunner,
helper pipeline.PluginHelper) error {
if runner.Encoder() == nil {
return errors.New("Encoder required.")
}
output.runner = runner
// Register callbacks to handle events
registerCallbacks(output)
var err error
// Connect to the Irc Server
err = output.Conn.Connect(output.Server)
if err != nil {
return fmt.Errorf("Unable to connect to irc server %s: %s",
output.Server, err)
}
// Start a goroutine for recieving messages, and throttling before sending
// to the Irc Server
output.wg.Add(1)
go processOutQueue(output)
var outgoing []byte
ok := true
inChan := runner.InChan()
var pack *pipeline.PipelinePack
for ok {
select {
case pack, ok = <-inChan:
case <-output.die:
ok = false
}
if !ok {
break
}
outgoing, err = runner.Encode(pack)
if err != nil {
output.runner.LogError(err)
} else if outgoing != nil {
// Send the message to each irc channel. If the out queue is full,
// then we need to drop the message and log an error.
for i, ircChannel := range output.Channels {
ircMsg := IrcMsg{outgoing, ircChannel, i}
select {
case output.OutQueue <- ircMsg:
default:
output.runner.LogError(ErrOutQueueFull)
}
}
}
pack.Recycle()
}
output.cleanup()
return nil
}
func (cwo *CloudwatchOutput) Submitter(payloads chan CloudwatchDatapoints,
or pipeline.OutputRunner) {
var (
payload CloudwatchDatapoints
curTry int
backOff time.Duration = time.Duration(10) * time.Millisecond
err error
stopping bool
)
curDuration := backOff
for !stopping {
select {
case stopping = <-cwo.stopChan:
continue
case payload = <-payloads:
for curTry < cwo.retries {
_, err = cwo.cw.PutMetricData(payload.Datapoints)
if err != nil {
curTry += 1
time.Sleep(curDuration)
curDuration *= 2
} else {
break
}
}
curDuration = backOff
curTry = 0
if err != nil {
or.LogError(err)
err = nil
}
}
}
close(cwo.stopChan)
}
// Run is the plugin's main loop
//iterates over received messages, checking against
//message hostname and delivering to the output if hostname is in our config.
func (o *TwilioOutput) Run(runner pipeline.OutputRunner, helper pipeline.PluginHelper) (
err error) {
var (
to, sms string
exc *gotwilio.Exception
)
for pack := range runner.InChan() {
sms = fmt.Sprintf("%s [%d] %s@%s: %s",
utils.TsTime(pack.Message.GetTimestamp()).Format(time.RFC3339),
pack.Message.GetSeverity(), pack.Message.GetLogger(),
pack.Message.GetHostname(), pack.Message.GetPayload())
pack.Recycle()
for _, to = range o.To {
_, exc, err = o.client.SendSMS(o.From, to, sms, "", "")
if err == nil && exc != nil {
return fmt.Errorf("%s: %d\n%s", exc.Message, exc.Code, exc.MoreInfo)
}
}
}
return
}
func (o *UnixOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
if or.Encoder() == nil {
return errors.New("Encoder required.")
}
var (
outBytes []byte
e error
)
for pack := range or.InChan() {
if outBytes, e = or.Encode(pack); e != nil {
or.LogError(fmt.Errorf("Error encoding message: %s", e.Error()))
} else if outBytes != nil {
o.conn.Write(outBytes)
}
pack.Recycle()
}
return
}
func (o *OpenTsdbOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
if or.Encoder() == nil {
return errors.New("Encoder must be specified.")
}
var (
e error
outBytes []byte
)
inChan := or.InChan()
for i := 0; i < o.TsdbWriterCount; i++ {
go WriteDataToOpenTSDB(o)
}
for pack := range inChan {
outBytes, e = or.Encode(pack)
pack.Recycle(e)
if e != nil {
or.LogError(e)
continue
}
if outBytes == nil {
continue
}
if e != nil {
log.Printf("OpenTsdbOutput-%s", e.Error())
continue
}
//fmt.Printf("OpenTsdbOutput-165-%v", logMsg)
o.logMsgChan <- outBytes
//fmt.Println("OpenTsdbOutput:", string(outBytes))
}
return
}
func (no *NsqOutput) Run(or pipeline.OutputRunner, h pipeline.PluginHelper) (err error) {
var (
encoder client.Encoder
msg *message.Message
msgBody []byte = make([]byte, 0, 1024)
pack *pipeline.PipelinePack
)
conf := no.conf
encoder = client.NewProtobufEncoder(nil)
for pack = range or.InChan() {
if conf.Serialize {
msg = pack.Message
if err = encoder.EncodeMessageStream(msg, &msgBody); err != nil {
or.LogError(err)
err = nil
pack.Recycle()
continue
}
//err := no.nsqwriter.PublishAsync(conf.Topic, []byte(pack.Message.GetPayload()), nil)
//err = no.nsqwriter.PublishAsync(conf.Topic, msgBody, nil)
_, _, err = no.nsqwriter.Publish(conf.Topic, msgBody)
if err != nil {
or.LogError(fmt.Errorf("error in writer.PublishAsync"))
}
msgBody = msgBody[:0]
} else {
err = no.nsqwriter.PublishAsync(conf.Topic, []byte(pack.Message.GetPayload()), nil)
if err != nil {
or.LogError(fmt.Errorf("error in writer.PublishAsync"))
}
}
pack.Recycle()
}
return
}